Python3 concurrentfutures — 管理并发任务池

import concurrent.futures
import urllib.request

# 定义一个函数，下载指定URL的网页内容
def download_website(url):
    with urllib.request.urlopen(url) as response:
        return response.read()

# 定义URL列表
urls = [
    'https://www.python.org',
    'https://www.python.org/about/',
    'https://www.onlamp.com/pub/a/python/2003/04/16/metaclasses.html',
    # ... 添加更多的URLs
]

# 使用concurrent.futures.ThreadPoolExecutor来并发下载网页
def download_websites(urls):
    with concurrent.futures.ThreadPoolExecutor() as executor:
        future_to_url = {executor.submit(download_website, url): url for url in urls}
        for future in concurrent.futures.as_completed(future_to_url):
            url = future_to_url[future]
            try:
                data = future.result()
                print(f'{url} page is {len(data)} bytes')
            except Exception as exc:
                print(f'{url} generated an exception: {exc}')

# 运行并发下载
download_websites(urls)