diff --git a/internal/gitlab/gitlab.go b/internal/gitlab/gitlab.go index 84dfd3f..343b091 100644 --- a/internal/gitlab/gitlab.go +++ b/internal/gitlab/gitlab.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/go-git/go-git/v5" @@ -11,7 +12,14 @@ import ( "github.com/xanzy/go-gitlab" ) -const defProjectsPerPage = 30 +// Will determine number of total projects, +// then based on projectsPerPage (request) and +// projectsPerGoroutine, will spin off goroutines +// with offsets +const ( + projectsPerPage = 20 + projectsPerGoroutine = 200 +) type Client struct { Ctx context.Context @@ -154,38 +162,72 @@ func (c *Client) streamProjects(pi *ProgressInfo, ownerOnly bool) { listOpts := &gitlab.ListProjectsOptions{ ListOptions: gitlab.ListOptions{ - PerPage: defProjectsPerPage, + PerPage: projectsPerPage, Page: 1, }, Archived: gitlab.Ptr[bool](false), Owned: gitlab.Ptr[bool](ownerOnly), } - var numProjects int - for { - projects, resp, err := c.ListProjects(listOpts) + // Get total number of projects + projectsTotal := c.getTotalProjects(listOpts) + numGoroutines := projectsTotal / projectsPerGoroutine - if err != nil { - pi.ErrorChan <- err - break - } + wg := sync.WaitGroup{} + startPage := 1 + for i := 1; i <= numGoroutines+1; i++ { + wg.Add(1) + endPage := startPage + (projectsPerGoroutine / projectsPerPage) + go func(startPage int, endPage int) { + defer wg.Done() + opts := *listOpts + opts.Page = startPage + for { + projects, resp, err := c.ListProjects(&opts) - // We're done when we have it all or our context is done - if c.Ctx.Err() != nil || resp.NextPage == 0 { - pi.DoneChan <- nil - break - } + if err != nil { + pi.ErrorChan <- err + break + } - numProjects += len(projects) - pi.ProjectsChan <- projects - pi.ProgressChan <- Progress{ - Page: resp.CurrentPage, - Pages: resp.TotalPages, - Projects: numProjects, - TotalProjects: resp.TotalItems, - } - listOpts.Page = resp.NextPage + pi.ProjectsChan <- projects + pi.ProgressChan <- Progress{ + Page: resp.CurrentPage, + Pages: resp.TotalPages, + Projects: len(projects), + TotalProjects: resp.TotalItems, + } + + // We're done when we have it all or our context is done + // or we've hit our total pages + if c.Ctx.Err() != nil || resp.NextPage == 0 { + break + } else if opts.Page == endPage { + break + } + + opts.Page = resp.NextPage + } + }(startPage, endPage) + startPage = endPage + 1 } + wg.Wait() + pi.DoneChan <- nil +} + +func (c *Client) getTotalProjects(opts *gitlab.ListProjectsOptions) int { + reqOpts := *opts + reqOpts.ListOptions = gitlab.ListOptions{ + Page: 1, + PerPage: 1, + } + + var projects int + if _, r, e := c.gitlab.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil { + projects = r.TotalItems + } + + return projects } // Returns a list of projects along with the next page and an error diff --git a/internal/projects/projects.go b/internal/projects/projects.go index e8bfcf7..274dd18 100644 --- a/internal/projects/projects.go +++ b/internal/projects/projects.go @@ -146,7 +146,6 @@ func (c *Cache) LoadProjects() { defer pBar.Stop() - var curProjects int for { select { case p := <-progressInfo.ProgressChan: @@ -157,8 +156,7 @@ func (c *Cache) LoadProjects() { // This sucks, has to be a better way, and why is the logger incompatible // with the progressbar? pterm.Debug.Println(fmt.Sprintf("Update received: %#v", p)) - pBar.Add(p.Projects - curProjects) - curProjects = p.Projects + pBar.Add(p.Projects) case p := <-progressInfo.ProjectsChan: c.Projects = append(c.Projects, p...) case e := <-progressInfo.ErrorChan: @@ -167,7 +165,7 @@ func (c *Cache) LoadProjects() { c.log.Warn("LoadProjects cancelled", c.log.Args("reason", c.gitlab.Ctx.Err())) return case <-progressInfo.DoneChan: - pBar.Add(pBar.Total - curProjects) + // pBar.Add(pBar.Total - curProjects) fmt.Println("") c.log.Info("Project load complete") return