Scrapy：如果某些条件适用，如何重新下载已缓存的URL

我在scrapy的缓存后端中存储了大量http数据。某些页面包含错误数据。这些URL需要重新安排，以便在下次运行scrapy时下载

我提出了修改scrapy附带的虚拟缓存策略的想法。不幸的是，这似乎不起作用

有人能看到方法“是否缓存”\u“响应”\u“刷新”中的错误吗

---

我认为这里的答案是，你的内容很可能被压缩或压缩了

试一试

不能说这个解决方案是多功能的，但它很可能在您的情况下起作用

import os
import cPickle as pickle
from time import time
from weakref import WeakKeyDictionary
from email.utils import mktime_tz, parsedate_tz
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.request import request_fingerprint
from scrapy.utils.project import data_path
from scrapy.utils.httpobj import urlparse_cached


class DummyPolicy(object):

    def __init__(self, settings):
        self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
        self.ignore_http_codes = [int(x) for x in settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')]

    def should_cache_request(self, request):
        return urlparse_cached(request).scheme not in self.ignore_schemes

    def should_cache_response(self, response, request):
        return response.status not in self.ignore_http_codes

    def is_cached_response_fresh(self, response, request):
        if "thisstring" in response.body.lower():
            print "got mobile page. redownload"
            return False
        else:
            return True

    def is_cached_response_valid(self, cachedresponse, response, request):
        return True

from scrapy.utils.gz import gunzip
if "thisstring" in gunzip(response.body).lower()