探索分布式爬虫的新境界：Scrapy-Redis 深度解析

import redis
from scrapy.utils.project import get_project_settings

class RedisSpiderMiddleware:
    """Spider中间件，用于处理爬虫的Redis相关操作"""

    def __init__(self, server, key):
        self.server = server
        self.key = key
        self.priority = 1000

    @classmethod
    def from_crawler(cls, crawler):
        """使用爬虫设置初始化中间件"""
        settings = get_project_settings()
        server = redis.StrictRedis(host=settings['REDIS_HOST'],
                                   port=settings['REDIS_PORT'],
                                   db=settings['REDIS_DB'])
        key = settings.get('REDIS_START_URLS_KEY', 'scrapy:start_urls')
        return cls(server, key)

    def process_spider_open(self, spider):
        """爬虫开启时，从Redis中获取起始URLs"""
        start_urls = self.server.lrange(self.key, 0, -1)
        for url in start_urls:
            spider.crawler.engine.crawl(spider.make_requests_from_url(url), spider)

    def process_spider_output(self, response, result, spider):
        """爬虫产生输出时，将新的items和requests存储到Redis中"""
        for item in result:
            if isinstance(item, dict):
                # 将Item存储到Redis中
                pass
            elif isinstance(item, Request):
                # 将Request的callback和priority记录到Redis中
                pass
        return result

    def process_spider_exception(self, response, exception, spider):
        """爬虫异常处理"""
        # 异常处理逻辑
        pass

    def process_start_requests(self, start_requests, spider):
        """处理起始请求"""
        for req in start_requests:
            self.server.rpush(self.key, req.url)
            yield req