package gitlabremote import ( "sync" "github.com/xanzy/go-gitlab" "gitea.libretechconsulting.com/rmcguire/git-project-manager/internal/remotes/load" "gitea.libretechconsulting.com/rmcguire/git-project-manager/internal/remotes/remote" ) // Will determine number of total projects, // then based on projectsPerPage (request) and // projectsPerGoroutine, will spin off goroutines // with offsets const ( projectsPerPage = 20 projectsPerGoroutine = 200 ) func (r *GitlabRemote) StreamProjects(pi *load.ProgressInfo, opts *remote.RemoteQueryOpts) { defer close(pi.ProgressChan) defer close(pi.ProjectsChan) listOpts := *DefaultListOpts listOpts.Owned = gitlab.Ptr[bool](opts.OwnerOnly) // Get total number of projects numGoroutines := pi.NumProjects / projectsPerGoroutine wg := sync.WaitGroup{} startPage := 1 for i := 1; i <= numGoroutines+1; i++ { wg.Add(1) endPage := startPage + (projectsPerGoroutine / projectsPerPage) go func(startPage int, endPage int) { defer wg.Done() opts := listOpts opts.Page = startPage for { projects, resp, err := r.ListProjects(&opts) if err != nil { pi.ErrorChan <- err break } pi.ProjectsChan <- projects pi.ProgressChan <- load.Progress{ Page: resp.CurrentPage, Pages: resp.TotalPages, Projects: len(projects), TotalProjects: resp.TotalItems, } // We're done when we have it all or our context is done // or we've hit our total pages if r.info.Context().Err() != nil || resp.NextPage == 0 { break } else if opts.Page == endPage { break } opts.Page = resp.NextPage } }(startPage, endPage) startPage = endPage + 1 } wg.Wait() pi.DoneChan <- nil }