Python 使用scrapy登录网站

我在写一只蜘蛛。在其中，我试图通过登录一个网站来使用scraping来抓取该网站。我写了一个蜘蛛，但在登录网站时仍然遇到问题。我写了整个蜘蛛，但无法解决登录问题。请看一下我的代码

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import FormRequest
from scrapy.utils.response import open_in_browser

class ScotlandSpider(scrapy.Spider):
    name = 'scotland'
    allowed_domains = ['www.whoownsscotland.org.uk']
    login_url = r'http://www.whoownsscotland.org.uk/login.php?p=%2Fsearch.php'
    start_urls = ['http://www.whoownsscotland.org.uk/search.php']

    def login(self , response):
        data = {
            'name' : 'USERNAME',
            'pass' : 'PASSWORD',
            'previous' : r'%2Fsearch.php',
            'login' : 'login'
        }
        yield FormRequest(url=self.login_url, formdata=data ,callback=self.parse)

    def parse(self, response):
        open_in_browser(response)
        links = response.xpath('//p/a/@href').extract()
        for link in links:
            absoulute_url = response.urljoin(link)
            yield scrapy.Request(absoulute_url , callback=self.parse_links)

    def parse_links(self , response):
        cities = response.xpath('//*[@id="layout-right"]/table/tr/td/p/a/@href').extract()
        for city in cities:
            absoulute_url_new = response.urljoin(city)
            yield scrapy.Request(absoulute_url_new , callback=self.parse_cities)

    def parse_cities(self , response):
        record = response.xpath('//*[@id="layout-left"]/table/tr')
        estate =  record[0].xpath('.//td/text()').extract()
        courty =  record[1].xpath('.//td/text()').extract()
        grid_ref =  record[2].xpath('.//td/text()').extract()
        acreage =  record[3].xpath('.//td/text()').extract()
        os_15 =  record[4].xpath('.//td/text()').extract()
        owner  = record[5].xpath('.//td/text()').extract()
        owner_address = record[6].xpath('.//td/text()').extract()
        property_address = record[7].xpath('.//td/text()').extract()
        website  = record[8].xpath('.//td/text()').extract()
        further_info = record[9].xpath('.//td//text()').extract()
        contacts = record[10].xpath('.//td//text()').extract()
        regsiters_sheet = record[11].xpath('.//td//text()').extract()
        regsiters_certificate = record[12].xpath('.//td//text()').extract()
        currency_of_data = record[13].xpath('.//td//text()').extract()

        yield {
            "Estate" : estate,
            "County" : courty,
            "Grid Reference" : grid_ref,
            "Acreage" : acreage,
            "OS 1:50k Sheet" : os_15,
            "Owner" : owner,
            "Owner Address" : owner_address,
            "Property Address" : property_address,
            "Website" : website,
            "Further Information" : further_info,
            "Contacts" : contacts,
            "Registers of Scotland Sasines Search Sheet No" : regsiters_sheet,
            "Registers of Scotland Land Certificate No" : regsiters_certificate ,
            "Currency of Data" : currency_of_data
        }

---

问题很简单：您创建了

login（）

方法，但从未调用它

解决此问题的最简单方法是将该方法重命名为。

---

然后，scrapy将调用此方法来生成初始请求，而不是从

start\u URL

生成请求。您应该更改帐户中的密码，因为您刚刚发布了密码，从现在起，它将在编辑中可见。仍然面临问题，我是scrapy的新手，请您对我的代码进行必要的更改，以便我可以运行它。@haider阅读并对您的代码进行必要的更改。@haider方法名称是使登录生效所需的唯一更改。在你更改密码之前，我测试了这个。如果还有其他问题，可能需要一个新问题。