Python 爬虫爬取多页数据

import requests
from bs4 import BeautifulSoup
 
def get_soup(url, params=None):
    response = requests.get(url, params=params)
    if response.status_code == 200:
        return BeautifulSoup(response.text, 'html.parser')
    else:
        return None
 
def extract_data(soup):
    # 假设数据在<div class="item">中
    items = soup.find_all('div', class_='item')
    for item in items:
        # 提取数据并处理，例如打印
        print(item.get_text())
 
def crawl_pages(start_page, end_page, url, params_template):
    for page in range(start_page, end_page + 1):
        params = params_template.copy()
        params['page'] = page
        soup = get_soup(url, params)
        if soup:
            extract_data(soup)
 
# 示例使用
url = 'http://example.com/search'
params_template = {
    'query': 'python',
    'page': None
}
start_page = 1
end_page = 3
crawl_pages(start_page, end_page, url, params_template)

这个示例代码展示了如何使用Python的requests库和BeautifulSoup库来爬取多页数据。get_soup函数负责发送请求并获取页面的soup对象。extract_data函数用于从soup对象中提取数据。crawl_pages函数负责遍历页码，并将每一页的soup对象传递给extract_data函数进行数据提取。这个例子中的url和params_template需要根据实际的网站进行调整。