import redis
from scrapy.utils.project import get_project_settings
class RedisSpiderMiddleware:
"""Spider中间件，用于处理爬虫的Redis相关操作"""
def __init__(self, server, key):
self.server = server
self.key = key
self.priority = 1000
@classmethod
def from_crawler(cls, crawler):
"""使用爬虫设置初始化中间件"""
settings = get_project_settings()
server = redis.StrictRedis(host=settings['REDIS_HOST'],
port=settings['REDIS_PORT'],
db=settings['REDIS_DB'])
key = settings.get('REDIS_START_URLS_KEY', 'scrapy:start_urls')
return cls(server, key)
def process_spider_open(self, spider):
"""爬虫开启时，从Redis中获取起始URLs"""
start_urls = self.server.lrange(self.key, 0, -1)
for url in start_urls:
spider.crawler.engine.crawl(spider.make_requests_from_url(url), spider)
def process_spider_output(self, response, result, spider):
"""爬虫产生输出时，将新的items和requests存储到Redis中"""
for item in result:
if isinstance(item, dict):
# 将Item存储到Redis中
pass
elif isinstance(item, Request):
# 将Request的callback和priority记录到Redis中
pass
return result
def process_spider_exception(self, response, exception, spider):
"""爬虫异常处理"""
# 异常处理逻辑
pass
def process_start_requests(self, start_requests, spider):
"""处理起始请求"""
for req in start_requests:
self.server.rpush(self.key, req.url)
yield req这个示例代码展示了如何使用Redis来管理Scrapy爬虫的起始URLs和处理过程中产生的Items和Requests。它提供了一个基本框架，开发者可以根据实际需求进一步完善具体的存储逻辑。