C# 爬虫开发小结

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
 
namespace WebCrawlerDemo
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var crawler = new WebCrawler();
            await crawler.CrawlAsync("https://www.example.com");
            Console.WriteLine($"Crawled {crawler.CrawledUrls.Count} URLs.");
        }
    }
 
    public class WebCrawler
    {
        public HashSet<string> CrawledUrls { get; private set; } = new HashSet<string>();
        private HttpClient _httpClient;
 
        public WebCrawler()
        {
            _httpClient = new HttpClient();
        }
 
        public async Task CrawlAsync(string url)
        {
            await CrawlPageAsync(url);
            var pendingUrls = new Queue<string>(CrawledUrls);
 
            while (pendingUrls.Count > 0)
            {
                string pendingUrl = pendingUrls.Dequeue();
                await CrawlPageAsync(pendingUrl);
 
                // 提取链接并加入待爬取队列
                var pageLinks = ExtractLinks(pendingUrl);
                foreach (var link in pageLinks)
                {
                    if (!CrawledUrls.Contains(link) && IsInternalLink(link, url))
                    {
                        CrawledUrls.Add(link);
                        pendingUrls.Enqueue(link);
                    }
                }
            }
        }
 
        private async Task CrawlPageAsync(string url)
        {
            try
            {
                Console.WriteLine($"Crawling {url}");
                await _httpClient.GetAsync(url); // 这里只是获取页面，不保存内容
                CrawledUrls.Add(url);
            }
            catch (HttpRequestException e)
            {
                Console.WriteLine($"An error occurred: {e.Message}");
            }
        }
 
        private string[] ExtractLinks(string url)
        {
            // 这里应该实现从页面内容中提取链接的逻辑
            throw new NotImplementedException();
        }
 
        private bool IsInternalLink(string link, string baseUrl)
        {
            // 判断链接是否是内部链接的逻辑
            throw new NotImplementedException();
        }
    }
}

这个代码实例提供了一个简化的Web爬虫框架，展示了如何初始化一个爬虫，如何爬取一个页面，如何从页面中提取链接，以及如何通过队列管理待爬取的URL集合。虽然这个例子没有包含提取链接和判断内部链接的具体实现，因为这些通常涉及到HTML解析或正则表达式，但它展示了基本的爬虫工作流程。