package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/gocolly/colly/v2"
)

func main() {
	// Create a new collector
	c := colly.NewCollector(
		// Only visit domains in the allowed list
		colly.AllowedDomains("go.dev", "pkg.go.dev"),

		// Cache responses to avoid repeated requests
		// colly.CacheDir("./colly_cache"),

		// Limit parallel requests
		colly.Async(true),
	)

	// Limit request rate
	c.Limit(&colly.LimitRule{
		DomainGlob:  "*",
		Parallelism: 2,
		// Delay: 1 * time.Second,
	})

	// Called before making a request
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting:", r.URL.String())
	})

	// Called when response received
	c.OnResponse(func(r *colly.Response) {
		fmt.Printf("Response from %s: %d bytes\n", r.Request.URL, len(r.Body))
	})

	// Called when error occurs
	c.OnError(func(r *colly.Response, err error) {
		log.Printf("Error: %s - %v\n", r.Request.URL, err)
	})

	// Extract data using CSS selectors
	c.OnHTML("h1", func(e *colly.HTMLElement) {
		fmt.Println("H1:", e.Text)
	})

	// Extract links
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		text := strings.TrimSpace(e.Text)
		if text != "" && len(text) < 50 {
			fmt.Printf("Link: %s -> %s\n", text, link)
		}

		// Follow links (crawl)
		// e.Request.Visit(link)
	})

	// Extract structured data
	c.OnHTML("article", func(e *colly.HTMLElement) {
		title := e.ChildText("h1, h2")
		content := e.ChildText("p")
		fmt.Printf("Article: %s\n", title)
		if len(content) > 100 {
			content = content[:100] + "..."
		}
		fmt.Printf("Content: %s\n\n", content)
	})

	// Extract attributes
	c.OnHTML("img[src]", func(e *colly.HTMLElement) {
		src := e.Attr("src")
		alt := e.Attr("alt")
		fmt.Printf("Image: %s (alt: %s)\n", src, alt)
	})

	// Extract multiple elements
	c.OnHTML("ul.nav li", func(e *colly.HTMLElement) {
		fmt.Println("Nav item:", e.Text)
	})

	// Start scraping
	c.Visit("https://go.dev/")

	// Wait for async requests to complete
	c.Wait()

	// Clone collector for different settings
	c2 := c.Clone()
	c2.OnHTML("title", func(e *colly.HTMLElement) {
		fmt.Println("Title:", e.Text)
	})
	c2.Visit("https://pkg.go.dev/")
	c2.Wait()
}

// Advanced features:
//
// // Set custom headers
// c.OnRequest(func(r *colly.Request) {
//     r.Headers.Set("User-Agent", "MyBot/1.0")
//     r.Headers.Set("Accept-Language", "en-US")
// })
//
// // POST request with form data
// c.Post("https://example.com/login", map[string]string{
//     "username": "user",
//     "password": "pass",
// })
//
// // Handle cookies
// c.SetCookies("https://example.com", []*http.Cookie{
//     {Name: "session", Value: "abc123"},
// })
//
// // Use proxy
// c.SetProxy("http://proxy:8080")
//
// // Callbacks for finished
// c.OnScraped(func(r *colly.Response) {
//     fmt.Println("Finished scraping:", r.Request.URL)
// })
