package main import ( "context" "log" "os" "os/signal" "sync" "syscall" "time" "github.com/gammazero/workerpool" ) func startDownloadVerify(files map[string]string, lfiles map[string]string) (bool, []error) { sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) wp := workerpool.New(opt.Threads) completed := true // will be set to false by the functions below // Errors, mutex because workers report their own errors concurrently. errMutex := &sync.Mutex{} var errors []error errc := make(chan bool, opt.Threads) // used to signal an error occured, buffered in case multiple errors occured endc := make(chan bool, 1) // Thread to tell this goroutine to stop checking for signals go func() { for { select { case <-sigc: log.Println("Signal recieved, gracefully shutting down...") completed = false wp.Stop() return case <-errc: log.Println("Error occured, gracefully stopping...") completed = false wp.Stop() return case <-endc: return } } }() // We're going to pause so that all jobs will be submitted by the time // any errors will occur (when it starts) pctx, c := context.WithCancel(context.Background()) wp.Pause(pctx) for k, v := range files { var lv *string if _, ok := lfiles[k]; ok { s := lfiles[k] lv = &s } { // new scope since we're dealing with a closure // set variables within our scope to the outside one. k := k v := v lv := lv f := func() { err := downloadAndVerify(k, v, lv) if err != nil { errMutex.Lock() defer errMutex.Unlock() errors = append(errors, err) if len(errors) > 0 { completed = false errc <- true return } } } wp.Submit(f) } // end scope } // Unpause c() // New channel, we'll wait for a signal to go ahead and exit // the function donec := make(chan bool) go func() { // hack, I'm going to fork the worker library so that // we can wait for a worker pool to stop instead of finish // its queue using a function like WaitStop for { if wp.WaitingQueueSize() == 0 && !wp.Stopped() { wp.Stop() donec <- true return } if wp.Stopped() { donec <- true return } time.Sleep(8 * time.Millisecond) } }() <-donec return completed, errors }