【爬虫系列】爬取糗事百科--正则表达式（超详细）

import requests
import re

def get_content(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
        'Host': 'www.qiushibaike.com'
    }
    response = requests.get(url, headers=headers)
    return response.text

def parse_content(html):
    # 正则表达式解析出段子内容
    # 注意：这里的正则表达式可能需要根据实际情况进行调整
    pattern = re.compile('<div class="content">.*?<span>(.*?)</span>', re.S)
    items = re.findall(pattern, html)
    return items

def main(url):
    html = get_content(url)
    jokes = parse_content(html)
    for joke in jokes:
        print(joke)

if __name__ == '__main__':
    url = 'https://www.qiushibaike.com/text/'
    main(url)