Improve cache load by a shitload

internal/gitlab/gitlab.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/go-git/go-git/v5"
@@ -11,7 +12,14 @@ import (
 	"github.com/xanzy/go-gitlab"
 )
 
-const defProjectsPerPage = 30
+// Will determine number of total projects,
+// then based on projectsPerPage (request) and
+// projectsPerGoroutine, will spin off goroutines
+// with offsets
+const (
+	projectsPerPage      = 20
+	projectsPerGoroutine = 200
+)
 
 type Client struct {
 	Ctx    context.Context
@@ -154,38 +162,72 @@ func (c *Client) streamProjects(pi *ProgressInfo, ownerOnly bool) {
 
 	listOpts := &gitlab.ListProjectsOptions{
 		ListOptions: gitlab.ListOptions{
-			PerPage: defProjectsPerPage,
+			PerPage: projectsPerPage,
 			Page:    1,
 		},
 		Archived: gitlab.Ptr[bool](false),
 		Owned:    gitlab.Ptr[bool](ownerOnly),
 	}
 
-	var numProjects int
-	for {
-		projects, resp, err := c.ListProjects(listOpts)
+	// Get total number of projects
+	projectsTotal := c.getTotalProjects(listOpts)
+	numGoroutines := projectsTotal / projectsPerGoroutine
 
-		if err != nil {
-			pi.ErrorChan <- err
-			break
-		}
+	wg := sync.WaitGroup{}
+	startPage := 1
+	for i := 1; i <= numGoroutines+1; i++ {
+		wg.Add(1)
+		endPage := startPage + (projectsPerGoroutine / projectsPerPage)
+		go func(startPage int, endPage int) {
+			defer wg.Done()
+			opts := *listOpts
+			opts.Page = startPage
+			for {
+				projects, resp, err := c.ListProjects(&opts)
 
-		// We're done when we have it all or our context is done
-		if c.Ctx.Err() != nil || resp.NextPage == 0 {
-			pi.DoneChan <- nil
-			break
-		}
+				if err != nil {
+					pi.ErrorChan <- err
+					break
+				}
 
-		numProjects += len(projects)
-		pi.ProjectsChan <- projects
-		pi.ProgressChan <- Progress{
-			Page:          resp.CurrentPage,
-			Pages:         resp.TotalPages,
-			Projects:      numProjects,
-			TotalProjects: resp.TotalItems,
-		}
-		listOpts.Page = resp.NextPage
+				pi.ProjectsChan <- projects
+				pi.ProgressChan <- Progress{
+					Page:          resp.CurrentPage,
+					Pages:         resp.TotalPages,
+					Projects:      len(projects),
+					TotalProjects: resp.TotalItems,
+				}
+
+				// We're done when we have it all or our context is done
+				// or we've hit our total pages
+				if c.Ctx.Err() != nil || resp.NextPage == 0 {
+					break
+				} else if opts.Page == endPage {
+					break
+				}
+
+				opts.Page = resp.NextPage
+			}
+		}(startPage, endPage)
+		startPage = endPage + 1
 	}
+	wg.Wait()
+	pi.DoneChan <- nil
+}
+
+func (c *Client) getTotalProjects(opts *gitlab.ListProjectsOptions) int {
+	reqOpts := *opts
+	reqOpts.ListOptions = gitlab.ListOptions{
+		Page:    1,
+		PerPage: 1,
+	}
+
+	var projects int
+	if _, r, e := c.gitlab.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil {
+		projects = r.TotalItems
+	}
+
+	return projects
 }
 
 // Returns a list of projects along with the next page and an error