Python 从xml文件中删除URL并删除这些URL
因此,我在items.py文件中声明了一个名为ArtscraperItem的项Python 从xml文件中删除URL并删除这些URL,python,web-scraping,scrapy,scrapy-spider,Python,Web Scraping,Scrapy,Scrapy Spider,因此,我在items.py文件中声明了一个名为ArtscraperItem的项 import scrapy class ArtscraperItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() date = scrapy.Field() date_str = scrapy.Field() url = scrapy.Field()
import scrapy
class ArtscraperItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
date = scrapy.Field()
date_str = scrapy.Field()
url = scrapy.Field()
title = scrapy.Field()
art_content = scrapy.Field()
我正在运行这个spider并从xml文件收集数据。但是,我还需要从xml文件中获取URL,然后刮取这些URL以获取文章的内容,并将其添加为项目[art_content]。我在stackoverflow上看到了类似的东西,但他们没有使用以前声明的项,所以我不知道如何使用它。因此,我需要从我抓取的url中获取内容,并将其添加到我在解析方法中创建的ArtscraperItem中
提前谢谢
所讨论的方法和第二个parse_article方法,该方法应该刮取收集的url并返回文章内容
from datetime import datetime as dt
import scrapy
from ArtScraper.items import ArtscraperItem
class PostSpider(scrapy.Spider):
article = ""
name = 'crawly'
allowed_domains = ['bbc.com/arabic']
start_urls = ['http://feeds.bbci.co.uk/arabic/rss.xml']
def parse(self, response):
articles = response.xpath('//channel/item')
for article in articles:
item = ArtscraperItem()
item['date']= dt.today()
item['date_str'] = article.xpath('pubDate/text()').extract_first()
item['url'] = article.xpath('link/text()').extract_first()
item['title'] = article.xpath('title/text()').extract_first()
url = item['url']
yield scrapy.Request(url, callback=self.parse_article)
yield item
def parse_article(self, response):
pars = response.xpath("//div[@class='story-body']/div[@class='story-body__inner']/p/text()").extract()
article = '-'.join(pars)
yield{
'art_content': article
}
设置.py文件
#Settings.py
BOT_NAME = 'ArtScraper'
SPIDER_MODULES = ['ArtScraper.spiders']
NEWSPIDER_MODULE = 'ArtScraper.spiders'
# Crawl responsibly by identifying yourself (and your website) on the
user-agent
#USER_AGENT = 'ArtScraper (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default:
16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-
delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = .25
RANDOMIZE_DOWNLOAD_DELAY=True
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'ArtScraper.middlewares.ArtscraperSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-
middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'ArtScraper.middlewares.ArtscraperDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'ArtScraper.pipelines.MongoPipeline': 300,
}
MONGO_URI='localhost:27017'
MONGO_DATABASE='george'
#Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel
to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-
middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE =
'scrapy.extensions.httpcache.FilesystemCacheStorage'
pipelines.py文件
import logging
import pymongo
class MongoPipeline(object):
collection_name = 'articles'
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls, crawler):
## pull in information from settings.py
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE')
)
def open_spider(self, spider):
## initializing spider
## opening db connection
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
def close_spider(self, spider):
## clean up when spider is closed
self.client.close()
def process_item(self, item, spider):
## how to handle each post
self.db[self.collection_name].insert(dict(item))
logging.debug("Post added to MongoDB")
return item
这叫做请求链和元结转。
您有2个请求制作1个项目,因此您需要:
Request.meta
属性,如下所示:
def parse(self, response):
articles = response.xpath('//channel/item')
for article in articles:
item = ArtscraperItem()
...
yield scrapy.Request(
url,
callback=self.parse_article,
meta={'item': item}, # carry over our item
)
def parse_article(self, response):
# retrieve carried over item
item = response.meta['item']
pars = response.xpath("//div[@class='story-body']/div[@class='story-body__inner']/p/text()").extract()
item['art_content'] = '-'.join(pars)
yield item
在这种情况下,我是否还必须更改pipelines.py文件或settings.py文件,因为我在解析文件末尾生成了项,但现在生成的是scrapy.Request。很抱歉,如果这是一个有点不切实际的问题,但这对我来说有点新,而且你的方法仍然没有将文章的内容添加到我的数据库集合中。如果你需要的话,我已经在我的帖子中添加了设置和管道。提前谢谢。好的,我修改了pipelines.py文件,稍后我将在编辑中包含新的pipelines文件,但是您的方法非常有效。非常感谢。@Hadinouou澄清一下-scrapy管道不接收
请求
对象,只接收项目
对象<代码>请求正在生成的对象转到scrapy的下载程序并继续爬网。