Python大作业——爬虫，2024年Python网络编程总结篇

import requests
from bs4 import BeautifulSoup
import re
 
def get_html(url):
    """
    获取网页内容
    :param url: 网页链接
    :return: 网页内容
    """
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        response.encoding = response.apparent_encoding
        return response.text
    except Exception as e:
        print(f"获取网页内容出错：{e}")
 
def parse_html(html):
    """
    解析网页内容，提取需要的信息
    :param html: 网页内容
    :return: 提取的信息列表
    """
    soup = BeautifulSoup(html, 'html.parser')
    titles = soup.find_all('div', class_='title')
    infos = []
    for title in titles:
        match = re.search(r'<a href="(.*?)">(.*?)</a>', str(title))
        if match:
            info = {
                'url': match.group(1),
                'title': match.group(2)
            }
            infos.append(info)
    return infos
 
def main():
    url = "https://www.example.com"
    html = get_html(url)
    infos = parse_html(html)
    for info in infos:
        print(f"标题：{info['title']}，链接：{info['url']}")
 
if __name__ == '__main__':
    main()

这个代码示例展示了如何使用Python的requests库获取网页内容，使用BeautifulSoup进行网页解析，并使用正则表达式提取特定信息。代码简洁明了，注重实用性和教育意义。