【Python爬虫】网络爬虫：信息获取与合规应用

import requests
from bs4 import BeautifulSoup
import re
import time
 
def get_soup(url):
    """
    获取网页内容并返回BeautifulSoup对象
    """
    headers = {
        'User-Agent': 'Mozilla/5.0',
        'From': 'your_email@example.com' # 替换为你的邮箱
    }
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return BeautifulSoup(response.text, 'html.parser')
    except requests.RequestException:
        print(f"An error occurred while trying to retrieve {url}")
        time.sleep(5)
 
def extract_links(soup):
    """
    从BeautifulSoup对象中提取新闻链接
    """
    # 根据实际HTML结构修改选择器
    return [link['href'] for link in soup.select('a.news-title') if link.get('href') and re.match(r'^http', link['href'])]
 
def main():
    url = 'http://example.com/news' # 替换为你要爬取的新闻网站
    soup = get_soup(url)
    if soup:
        links = extract_links(soup)
        for link in links:
            print(link)
 
if __name__ == '__main__':
    main()

这段代码提供了一个简单的网络爬虫示例，用于从一个假设的新闻网站中提取新闻链接。在实际应用中，你需要替换'example.com/news'为你要爬取的实际网站，并确保选择器（如'a.news-title'）匹配目标网站的HTML结构。此外，记得遵守网站的robots.txt规则和法律法规，尊重网站版权和隐私政策。