[golang] quick review guide quickreview (x) -- goroutine pool

goroutine The stack of is small at the beginning of its life cycle , Maybe it's just 2KB, But it is not fixed , Can be increased or decreased as required . Although we can create many without brains goroutine To perform operations , But if something happens to the program ,goroutine May soar to occupy memory , Everything becomes uncontrollable , For example, we create by looping goroutine, When the cycle condition is satisfied , Create huge goroutine, In severe cases, the system will crash . The blogger is also through teacher Yang Xu TCP This problem was found in the port scanner .

1. Circular scanning

Don't use goroutine, Direct loop scan port .

package main

import (
 "fmt"
 "net"
 "time"
)

func main() {
 fmt.Println("TCP Port scan start ...")
 start := time.Now()
 for i := 1; i <= 20; i++ {
  address := fmt.Sprintf("192.168.0.109:%d", i)
  conn, err := net.Dial("tcp", address)
  if err != nil {
   fmt.Printf(" %s  close \n", address)
   continue
  }
  conn.Close()
  fmt.Printf(" %s  open \n", address)
 }

 elasped := time.Since(start) / 1e9
 fmt.Printf("\n  After %d second  \n", elasped)
}

Copy

TCP Port scan start ...
 192.168.0.109:1  close 
 192.168.0.109:2  close 
 192.168.0.109:3  close 
 192.168.0.109:4  close 
 192.168.0.109:5  close 
 192.168.0.109:6  close 
 192.168.0.109:7  close 
 192.168.0.109:8  close 
 192.168.0.109:9  close 
 192.168.0.109:10  close 
 192.168.0.109:11  close 
 192.168.0.109:12  close 
 192.168.0.109:13  close 
 192.168.0.109:14  close 
 192.168.0.109:15  close 
 192.168.0.109:16  close 
 192.168.0.109:17  close 
 192.168.0.109:18  close 
 192.168.0.109:19  close 
 192.168.0.109:20  close 

  After 40 second 

Copy

2. Concurrent scanning

utilize goroutine Concurrent scanning

package main

import (
 "fmt"
 "net"
 "sync"
 "time"
)

var wg sync.WaitGroup

func main() {
 fmt.Println("TCP Port scan start ...")
 start := time.Now()
 for i := 1; i <= 20; i++ {
  wg.Add(1)
  go func(i int) {
   defer wg.Done()
   address := fmt.Sprintf("192.168.0.109:%d", i)
   conn, err := net.Dial("tcp", address)
   if err != nil {
    fmt.Printf(" %s  close \n", address)
    return
   }
   conn.Close()
   fmt.Printf(" %s  open \n", address)
  }(i)
 }
 wg.Wait()
 elasped := time.Since(start) / 1e9
 fmt.Printf(" After %d second ", elasped)
}

Copy

TCP Port scan start ...
 192.168.0.109:4  close 
 192.168.0.109:6  close  
 192.168.0.109:11  close 
 192.168.0.109:7  close  
 192.168.0.109:8  close  
 192.168.0.109:14  close 
 192.168.0.109:3  close 
 192.168.0.109:20  close 
 192.168.0.109:16  close 
 192.168.0.109:12  close 
 192.168.0.109:1  close 
 192.168.0.109:17  close 
 192.168.0.109:18  close 
 192.168.0.109:13  close 
 192.168.0.109:9  close 
 192.168.0.109:10  close 
 192.168.0.109:5  close 
 192.168.0.109:15  close 
 192.168.0.109:19  close 
 192.168.0.109:2  close 
 After 2 second 

Copy

Even if the code is changed to full port 1~65535, Time is only 21s

package main

import (
 "fmt"
 "net"
 "sync"
 "time"
)

var wg sync.WaitGroup

func main() {
 fmt.Println("TCP Port scan start ...")
 start := time.Now()
 for i := 1; i <= 65535; i++ {
  wg.Add(1)
  go func(i int) {
   defer wg.Done()
   address := fmt.Sprintf("192.168.0.109:%d", i)
   conn, err := net.Dial("tcp", address)
   if err != nil {
    fmt.Printf(" %s  close \n", address)
    return
   }
   conn.Close()
   fmt.Printf(" %s  open \n", address)
  }(i)
 }
 wg.Wait()
 elasped := time.Since(start) / 1e9
 fmt.Printf("\n  After %d second  \n", elasped)
}

Copy

 ...
 192.168.0.109:39702  close 
 192.168.0.109:37282  close 
 192.168.0.109:39490  close 
 192.168.0.109:39795  close 
 192.168.0.109:39613  close 
 192.168.0.109:39707  close 
 192.168.0.109:39723  close 
 192.168.0.109:39732  close 
 192.168.0.109:39806  close 
 192.168.0.109:39682  close 
 192.168.0.109:39809  close 
 192.168.0.109:39663  close 
 192.168.0.109:40006  close 
 192.168.0.109:39712  close 
 192.168.0.109:39804  close 
 192.168.0.109:39731  close 
 192.168.0.109:39701  close 
 192.168.0.109:39810  close 
 192.168.0.109:39725  close 
 192.168.0.109:39805  close 
 192.168.0.109:39713  close 
 192.168.0.109:39489  close 
 192.168.0.109:39791  close 
 192.168.0.109:39640  close 
 192.168.0.109:39667  close 
 192.168.0.109:39661  close 

  After 21 second 

Copy

It can also be seen here that goroutine Do concurrent operations , It's really fast .

3.goroutine pool( pool )

This is done at the end of the previous section , It was created. 6w many times goroutine, If the value is larger ,100w,1 $ ( Of course IP The port number is not so large ), Just a bold experiment ： Then you can see the memory in the floating window that monitors the memory No, no, no, No It's going up , Memory quickly ran out , Until the system is stuck , Even blue screen .

To avoid that , We urgently need to create a mechanism similar to thread pool , To limit what we create goroutine The number of , And reuse . Create a fixed number of m individual goroutine, utilize channel The mechanism of , Go to channel In the transfer n Data , Then assign to this m individual goroutine,m<=n.

It is also a port scanning task , The code transformation is as follows ：

package main

import (
 "fmt"
 "net"
)

func main() {
 fmt.Println("TCP Port scan start ...")
 fmt.Println(" The following ports are open :")
 n := make(chan int, 100)
 results := make(chan int, 100)
    
    // establish 20000 individual goroutine
 for m := 0; m < 20000; m++ {
  go worker(n, results)
 }
    
    //channel Represents the number of tasks 
 for i := 1; i < 65535; i++ {
  n <- i
 }
    close(n)
    
    // result 
 for port := range results {
  fmt.Println(port)
 }

}

func worker(ports <-chan int, results chan<- int) {
 for port := range ports {
  address := fmt.Sprintf("192.168.0.109:%d", port)
  conn, err := net.Dial("tcp", address)
  if err != nil {
   // fmt.Printf(" %s  close \n", address)
   continue
  }
  conn.Close()
  // fmt.Printf(" %s  open \n", address)
  results <- port
 }
}

Copy

TCP Port scan start ...
 The following ports are open :
80
81   
139  
902  
443  
445  
135  
912  
1433 
2383 
2179 
3306 
5357 
5040 
7680 
8080 
33060
43094
43095
49672
49664
49667
49666
49670
49665
50272

Copy