探秘FWC1994的Python Crawler：数据抓取的新纪元

import requests
from bs4 import BeautifulSoup
 
def crawl_site(url):
    """
    抓取指定URL的网页内容
    """
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        else:
            print(f"Failed to retrieve the webpage: {response.status_code}")
            return None
    except requests.exceptions.RequestException as e:
        print(f"An error occurred while crawling the webpage: {e}")
        return None
 
def parse_data(html):
    """
    解析网页内容，提取有效数据
    """
    soup = BeautifulSoup(html, 'html.parser')
    # 假设我们要提取所有的段落文本
    paragraphs = soup.find_all('p')
    return [p.get_text() for p in paragraphs]
 
def main():
    url = "https://example.com"  # 替换为你要抓取的网址
    html = crawl_site(url)
    if html:
        data = parse_data(html)
        for paragraph in data:
            print(paragraph)
 
if __name__ == "__main__":
    main()

这段代码展示了如何使用Python的requests库和BeautifulSoup库来实现一个简单的网页抓取和数据解析的例子。代码中的crawl_site函数用于获取网页内容，而parse_data函数用于从网页内容中提取数据。main函数则是程序的入口点，它调用这两个函数来完成整个抓取和解析的过程。