爬虫 | Python爬取微博实时热搜榜信息

import requests
from bs4 import BeautifulSoup
import time
import random
 
# 设置请求头，模拟浏览器访问
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
 
def get_weibo_hot_search(url):
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        return response.text
    return None
 
def parse_weibo_hot_search(html):
    soup = BeautifulSoup(html, 'lxml')
    hot_search_list = soup.find_all(class_='td-01')
    for hot_search in hot_search_list:
        rank = hot_search.find('em').text
        keyword = hot_search.find('a').text
        print(f'排名: {rank}, 关键词: {keyword}')
 
def main():
    url = 'https://s.weibo.com/top/summary'
    html = get_weibo_hot_search(url)
    parse_weibo_hot_search(html)
 
if __name__ == '__main__':
    main()

这段代码首先定义了请求头，模拟浏览器访问，然后定义了获取微博实时热搜榜页面源码的函数get_weibo_hot_search。接着定义了解析源码并提取关键词和排名的函数parse_weibo_hot_search。最后在main函数中调用这两个函数，完成微博实时热搜榜信息的爬取和解析。