基于Go 1.19的站点模板爬虫

package main

import (
    "fmt"
    "log"
    "net/http"
    "os"
    "time"

    "go.uber.org/zap"
    "go.uber.org/zap/zapcore"
    "golang.org/x/net/html"
)

// 初始化日志配置
func initLogger() (*zap.Logger, error) {
    encoderConfig := zapcore.EncoderConfig{
        TimeKey:        "ts",
        LevelKey:       "level",
        NameKey:        "logger",
        CallerKey:      "caller",
        MessageKey:     "msg",
        StacktraceKey:  "stacktrace",
        LineEnding:     zapcore.DefaultLineEnding,
        EncodeLevel:    zapcore.LowercaseLevelEncoder,
        EncodeTime:     zapcore.EpochMillisTimeEncoder,
        EncodeDuration: zapcore.SecondsDurationEncoder,
    }
    core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), os.Stdout, zapcore.InfoLevel)
    return zap.New(core), nil
}

// 爬取指定URL的页面内容
func crawlPage(logger *zap.Logger, url string) (string, error) {
    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Get(url)
    if err != nil {
        logger.Error("Error fetching URL", zap.Error(err), zap.String("url", url))
        return "", err
    }
    defer resp.Body.Close()

    doc, err := html.Parse(resp.Body)
    if err != nil {
        logger.Error("Error parsing HTML", zap.Error(err), zap.String("url", url))
        return "", err
    }

    return doc.Text(), nil
}

func main() {
    logger, err := initLogger()
    if err != nil {
        log.Fatalf("无法初始化日志: %v", err)
    }
    defer logger.Sync()

    url := "https://example.com"
    content, err := crawlPage(logger, url)
    if err != nil {
        logger.Error("爬取页面失败", zap.Error(err), zap.String("url", url))
        return
    }

    fmt.Printf("爬取的页面内容: %s\n", content)
}