Go语言Colly爬虫问题：收集非允许域名与单行输出优化

问题1：无法收集非允许域名到garbage数组 #

核心原因 #

1. colly.Collector的AllowedDomains是私有字段，不能直接通过c.AllowedDomains[...]访问，需使用官方提供的c.AllowedDomain(hostname)方法判断域名合法性。
2. 变量名拼写错误：garbeg应为garbage。
3. 未对garbage数组加锁，并发环境下会出现数据竞争。
4. 判断逻辑位置错误，应该在处理链接的最开始就检查域名是否合法。

修正逻辑 #

在处理href和src链接的回调最开始，先将相对链接转为绝对URL，解析域名后用AllowedDomain方法判断，不合法则加入garbage数组并跳过后续处理，同时加锁保护数组的并发写入。

问题2：统计信息重复多行输出 #

核心原因 #

1. 输出语句末尾加了\n换行符，导致每次输出都换行，\r仅能回到行首但无法清除之前的内容。
2. 并发环境下多个OnRequest回调同时输出，会导致输出混乱。

修正逻辑 #

新增互斥锁保护统计输出，同时使用\033[K清除行尾内容，确保每次输出都覆盖当前行，避免多行重复。

完整修正代码 #

package main

import (
    "encoding/json"
    "fmt"
    "log"
    "net/url"
    "os"
    "strings"
    "sync"
    "time"

    "github.com/gocolly/colly"
)

func main() {
    startURL := "https://example.com"    // Starting URL
    maxDepth := 1                         // Maximum depth to crawl
    maxThreads := 100                    // Maximum number of threads to use

    var wg sync.WaitGroup            // WaitGroup to synchronize threads
    var mu sync.Mutex                // Mutex to protect visited, endpointList and garbage
    var statsMu sync.Mutex           // Mutex to protect stats output

    visited := make(map[string]bool) // Keep track of visited URLs
    var endpointList []string
    var garbage []string
    // Create a new Collector with a limit on the maximum number of threads
    c := colly.NewCollector(
        //colly.Async(true),
        colly.AllowedDomains("example.com", "*.example.com"),
        colly.MaxDepth(maxDepth),
        colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"),
    )

    c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: maxThreads})

    // Set a random delay between requests to make them look more like human behavior
    c.Limit(&colly.LimitRule{
        RandomDelay: 5 * time.Second,
    })

    // Create a log file to save errors
    logFile, err := os.OpenFile("log.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
    if err != nil {
        log.Fatal(err)
    }
    defer logFile.Close()

    // Use the log file to log errors
    logger := log.New(logFile, "error: ", log.Ldate|log.Ltime|log.Lshortfile)

    // On every visited page, process href links
    c.OnHTML("[href]", func(e *colly.HTMLElement) {
        link := e.Attr("href")
        if strings.HasPrefix(link, "#") || 
            strings.HasPrefix(link, "tel") || 
            strings.HasPrefix(link, "mailto") || 
            strings.HasPrefix(link, "data:image") || 
            strings.HasPrefix(link, "javascript:") || 
            link == "" {
                return
        }
        
        // Check if domain is allowed, add to garbage if not
        absLink := e.Request.AbsoluteURL(link)
        if absLink != "" {
            u, err := url.Parse(absLink)
            if err == nil {
                mu.Lock()
                if !c.AllowedDomain(u.Hostname()) {
                    garbage = append(garbage, absLink)
                    mu.Unlock()
                    return
                }
                mu.Unlock()
            }
        }
        
        mu.Lock()
        defer mu.Unlock()
        if !visited[link] {
            visited[link] = true
            endpointList = append(endpointList, link)
            wg.Add(1)
            go func(absLink string) {
                defer wg.Done()
                if absLink != "" {
                    err := c.Visit(absLink)
                    if err != nil {
                        logger.Printf("%s (%s)>> %v\n", absLink, link, err)
                    }
                }
            }(e.Request.AbsoluteURL(link))
        }else if !strings.HasPrefix(link, "http") {
            linkURL := e.Request.AbsoluteURL(link)
            if linkURL != "" && !visited[linkURL] {
                visited[linkURL] = true
                endpointList = append(endpointList, linkURL)
                wg.Add(1)
                go func(linkURL string) {
                    defer wg.Done()
                    err := c.Visit(linkURL)
                    if err != nil {
                        logger.Printf("%s (%s)>> %v\n", linkURL, link, err)
                    }
                }(linkURL)
            }
        }
    })
    
    // On every visited page, process src links
    c.OnHTML("[src]", func(e *colly.HTMLElement) {
        jsURL := e.Attr("src")
        // Check if domain is allowed, add to garbage if not
        absURL := e.Request.AbsoluteURL(jsURL)
        if absURL != "" {
            u, err := url.Parse(absURL)
            if err == nil {
                mu.Lock()
                if !c.AllowedDomain(u.Hostname()) {
                    garbage = append(garbage, absURL)
                    mu.Unlock()
                    return
                }
                mu.Unlock()
            }
        }
        
        mu.Lock()
        defer mu.Unlock()
        if !visited[jsURL] {
            visited[jsURL] = true
            endpointList = append(endpointList, jsURL)
        }
    })
    
    // On every request, output crawling statistics
    var reqCount int
    var st time.Time
    c.OnRequest(func(r *colly.Request) {
        statsMu.Lock()
        defer statsMu.Unlock()
        
        reqCount++
        if reqCount == 1 {
            st = time.Now()
        }

        elapsed := time.Since(st)
        rps := 0
        if elapsed.Seconds() > 0 {
            rps = reqCount / int(elapsed.Seconds())
        } else {
            rps = reqCount
        }
        // Use \033[K to clear line tail, ensure single line update
        fmt.Printf("\r%d req/sec  %d endpoint(s) %d depth %v time passed\033[K", 
            rps, len(endpointList), r.Depth, elapsed.Truncate(time.Second))
    })

    // Start crawling from the initial URL
    visited[startURL] = true
    startTime := time.Now()
    c.Visit(startURL)

    // Wait for all threads to finish
    wg.Wait()

    // End stats output with a newline
    fmt.Println()
    
    // Final statistics
    fmt.Printf("\nCollected %d unique links\n", len(visited))
    duration := time.Since(startTime)
    rps := float64(len(visited)) / duration.Seconds()
    fmt.Printf("Crawled at %.2f requests per second\n", rps)

    // Save results to JSON
    results := struct {
        Endpoints []string `json:"endpoints"`
        Garbage   []string `json:"garbage"`
    }{
        Endpoints: endpointList,
        Garbage:   garbage,
    }

    file, err := os.Create("results.json")
    if err != nil {
        log.Fatal(err)
    }
    defer file.Close()

    encoder := json.NewEncoder(file)
    encoder.SetIndent("", "    ")
    err = encoder.Encode(results)
    if err != nil {
        log.Fatal(err)
    }
}

内容的提问来源于stack exchange，提问作者Khaled Kamal Ebrahim Ebrahim