Python 爬行但不刮
我试图使用Scrapy在以下网站()上进行抓取,我看到该页面被抓取,但没有返回任何项目。一切都与刮擦壳一起工作 以下是我的代码:Python 爬行但不刮,python,web-scraping,scrapy,web-crawler,Python,Web Scraping,Scrapy,Web Crawler,我试图使用Scrapy在以下网站()上进行抓取,我看到该页面被抓取,但没有返回任何项目。一切都与刮擦壳一起工作 以下是我的代码: class LeadHomeSpider(scrapy.Spider): name = "lead_home" start_urls = [ 'https://www.leadhome.co.za/search/property-for-sale/western-cape/4?sort=date', ]
class LeadHomeSpider(scrapy.Spider):
name = "lead_home"
start_urls = [
'https://www.leadhome.co.za/search/property-for-sale/western-cape/4?sort=date',
]
# parse search page
def parse(self, response):
# follow property link
offering = 'buy' if 'sale' in response.css('h1::text').get() else 'rent'
for prop in response.css('div.search__PropertyCardWrapper-sc-1j5dndx-0.bsqBpI'):
link = 'https://www.leadhome.co.za' + prop.css('a::attr(href)').get()
a = prop.css('p.styles__Label-h53xsw-16.bcSkCI::text').getall()
#prop_type = attempt_get_property_type(a[0]) if len(a) != 0 else None
area = a[1] if len(a) > 1 else None
yield scrapy.Request(
link,
meta={'item': {
'agency': self.name,
'url': link,
'area': area,
'offering': offering,
#'property_type': prop_type,
}},
callback=self.parse_property,
)
# follow to next page
next_page_number = response.xpath(
'//a[contains(@class, "styles__PageNumber-zln67a-0 jRCKhp")]/following-sibling::a/text()').get()
if next_page_number is not None:
new_page_link = 'https://www.leadhome.co.za/search/property-for-sale/western-cape/4?sort=date&page=' + next_page_number
next_page = response.urljoin(new_page_link)
yield scrapy.Request(next_page, callback=self.parse)
# parse property
def parse_property(self, response):
item = response.meta.get('item')
item['parking'] = response.xpath('//p[contains(text(), "Uncovered Parking:")]/following-sibling::p/text()').get()
...
你知道这里可能出了什么问题吗?欢迎提出任何建议!提前谢谢你 在CSS表达式中使用的是随机类值(
1j5dndx-0.bsqBpI
等),这就是代码不起作用的原因。下面是相同的代码,但使用XPath的包含
来匹配类的一部分:
def parse(self, response):
# follow property link
offering = 'buy' if 'sale' in response.css('h1::text').get() else 'rent'
# for prop in response.css('div.search__PropertyCardWrapper-sc-1j5dndx-0.bsqBpI'):
for prop in response.xpath('//div[contains(@class, "search__PropertyCardWrapper-sc-")]'):
link = prop.xpath('.//a/@href').get()
# a = prop.css('p.styles__Label-h53xsw-16.bcSkCI::text').getall()
prop_type = prop.xpath('(.//p[contains(@class, "styles__Label-")])[1]/text()').get()
# area = a[1] if len(a) > 1 else None
link = response.urljoin(link)
yield scrapy.Request(
url=link,
meta={'item': {
'agency': self.name,
'url': link,
# 'area': area,
'offering': offering,
'property_type': prop_type,
}},
callback=self.parse_property,
)