Scrapy爬虫开发流程全解析

scrapy startproject myproject

import scrapy

class MyItem(scrapy.Item):
    # 定义字段
    name = scrapy.Field()
    price = scrapy.Field()
    stock = scrapy.Field()

import scrapy
from myproject.items import MyItem

class MySpider(scrapy.Spider):
    name = 'my_spider'
    start_urls = ['http://example.com/']

    def parse(self, response):
        for item in response.css('div.product'):
            product = MyItem()
            product['name'] = item.css('div.name ::text').extract_first()
            product['price'] = item.css('div.price ::text').extract_first()
            product['stock'] = item.css('div.stock ::text').extract_first()
            yield product

        next_page_url = response.css('div.pager a.next ::attr(href)').extract_first()
        if next_page_url is not None:
            yield response.follow(next_page_url, self.parse)

class MyPipeline(object):
    def process_item(self, item, spider):
        # 处理item，例如保存到数据库
        return item

ITEM_PIPELINES = {
    'myproject.pipelines.MyPipeline': 300,
}

scrapy crawl my_spider