【爬虫实战】2024知乎热榜可视化爬取

import requests
from bs4 import BeautifulSoup
import pandas as pd
import matplotlib.pyplot as plt
 
# 设置请求头，模拟浏览器访问
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    'Accept-Encoding': 'gzip, deflate',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Upgrade-Insecure-Requests': '1'
}
 
# 请求网页
def get_html(url):
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except requests.RequestException:
        return None
 
# 解析网页，提取热榜数据
def parse_html(html):
    soup = BeautifulSoup(html, 'lxml')
    ranks = soup.find_all(class_='HotItemList-itemTitle')
    rank_list = []
    for rank in ranks:
        rank_list.append(rank.text)
    return rank_list
 
# 保存数据
def save_data(data, filename):
    with open(filename, 'w', encoding='utf-8') as f:
        for item in data:
            f.write(item + '\n')
 
# 读取数据
def read_data(filename):
    with open(filename, 'r', encoding='utf-8') as f:
        data = f.readlines()
    return [item.strip() for item in data]
 
# 可视化热榜数据
def visualize(data):
    plt.figure(figsize=(20, 10))
    plt.title('2024年知乎热榜', fontsize=20)
    plt.xlabel('排名', fontsize=16)
    plt.ylabel('话题', fontsize=16)
    plt.tick_params(labelsize=16)
    plt.barh(range(len(data)), list(data.values()), align='center')
    plt.yticks(list(data.keys()), list(data.values()))
    plt.show()
 
# 主函数
def main():
    url = 'https://www.zhihu.com/hot'
    html = get_html(url)
    rank_list = parse_html(html)
    # save_data(rank_list, 'zhihu_hot_topics.txt')
    # rank_list = read_data('zhihu_hot_topics.txt')
    visualize(rank_list)
 
if __name__ == '__main__':
    main()

这段代码实现了知乎热榜的爬取和可视化。首先，我们设置了请求头，以模拟浏览器访问知乎网站。然后，我们定义了get_html函数来发送请求并获取网页内容。parse_html函数用于解析网页，提取热榜话题并返回列表。最后，我们定义了save_data和read_data函数来保存和读取数据，以及visualize函数来绘制热榜的水平条形图，并在主函数main中调用这些函数。