Python网页爬虫爬取豆瓣Top250电影数据——Xpath数据解析_爬虫电影

import requests
from lxml import etree
import csv
import time
 
# 设置请求头，模拟浏览器访问
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
 
def get_data(url):
    # 发送请求，获取响应
    response = requests.get(url, headers=headers)
    return response.text
 
def parse_data(html):
    # 使用Xpath解析网页
    html = etree.HTML(html)
    # 影片名称
    name = html.xpath('//div[@class="info"]/div[@class="hd"]/a/span[1]/text()')
    # 影评分数
    score = html.xpath('//div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')
    # 评分人数
    people = html.xpath('//div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[4]/text()')
    # 上映日期
    time = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 导演
    director = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 编剧
    writer = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 类型
    type = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 区域
    area = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 语言
    language = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 上映时间
    time_show = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 集数
    part = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 国家
    country = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
    # 简介
    introduce = html.xpath('//div[@class="info"]/div[@class="bd"]/p[@class=""]/text()')
 
    return name, score, people, time, director, writer, type, area, language, time_show, part, country, introduce
 
def save_data(data):
    # 保存数据到CSV文件
    with open('douban_top250.csv', 'a', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(data)