Go crawler framework -colly actual combat (II) -- Douban top250 crawling

Original link :Hzy Blog

1. Try to use it today colly Come and crawl for the watercress Top 250！( Everyone likes to practice with him …)

Go straight to the code , There are notes on it .

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
	"regexp"
	"strings"
	"time"
)

func main() {
	t := time.Now()
	number := 1
	c := colly.NewCollector(func(c *colly.Collector) {
		extensions.RandomUserAgent(c) //  Set random header 
		c.Async=true
	},
		// Filter url, It's not https://movie.douban.com/top250?start=0&filter=  Of url
		colly.URLFilters(
			regexp.MustCompile("^(https://movie\\.douban\\.com/top250)\\?start=[0-9].*&filter="),
		),
	) //  Create collector 
	//  The format of the response is HTML, Extract the links in the page 
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		//fmt.Printf("find link: %s\n", e.Request.AbsoluteURL(link))
		c.Visit(e.Request.AbsoluteURL(link))
	})
	//  Get movie information 
	c.OnHTML("div.info", func(e *colly.HTMLElement) {
		e.DOM.Each(func(i int, selection *goquery.Selection) {
			movies := selection.Find("span.title").First().Text()
			director := strings.Join(strings.Fields(selection.Find("div.bd p").First().Text()), " ")
			quote := selection.Find("p.quote span.inq").Text()
			fmt.Printf("%d --> %s:%s %s\n", number, movies, director, quote)
			number += 1
		})
	})
	c.OnError(func(response *colly.Response, err error) {
		fmt.Println(err)
	})
	c.Visit("https://movie.douban.com/top250?start=0&filter=")
	c.Wait()
	fmt.Printf(" Spend time :%s",time.Since(t))
}

github Address :github Address

I think it is very convenient to use this framework , Tomorrow, I will try to crawl some websites that need to log in ！