Start moving gitlab code to remote interface

internal/remotes/gitlab/gitlab_strean.go

@@ -0,0 +1,78 @@
+package gitlabremote
+
+import (
+	"sync"
+
+	"github.com/xanzy/go-gitlab"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
+)
+
+// Will determine number of total projects,
+// then based on projectsPerPage (request) and
+// projectsPerGoroutine, will spin off goroutines
+// with offsets
+const (
+	projectsPerPage      = 20
+	projectsPerGoroutine = 200
+)
+
+var DefaultListOpts = &gitlab.ListProjectsOptions{
+	ListOptions: gitlab.ListOptions{
+		PerPage: projectsPerPage,
+		Page:    1,
+	},
+	Archived: gitlab.Ptr[bool](false),
+}
+
+func (r *GitlabRemote) StreamProjects(pi *load.ProgressInfo, opts *remote.RemoteQueryOpts) {
+	defer close(pi.ProgressChan)
+	defer close(pi.ProjectsChan)
+
+	listOpts := *DefaultListOpts
+	listOpts.Owned = gitlab.Ptr[bool](opts.OwnerOnly)
+
+	// Get total number of projects
+	numGoroutines := pi.NumProjects / projectsPerGoroutine
+
+	wg := sync.WaitGroup{}
+	startPage := 1
+	for i := 1; i <= numGoroutines+1; i++ {
+		wg.Add(1)
+		endPage := startPage + (projectsPerGoroutine / projectsPerPage)
+		go func(startPage int, endPage int) {
+			defer wg.Done()
+			opts := listOpts
+			opts.Page = startPage
+			for {
+				projects, resp, err := r.ListProjects(&opts)
+
+				if err != nil {
+					pi.ErrorChan <- err
+					break
+				}
+
+				pi.ProjectsChan <- projects
+				pi.ProgressChan <- load.Progress{
+					Page:          resp.CurrentPage,
+					Pages:         resp.TotalPages,
+					Projects:      len(projects),
+					TotalProjects: resp.TotalItems,
+				}
+
+				// We're done when we have it all or our context is done
+				// or we've hit our total pages
+				if r.info.Ctx.Err() != nil || resp.NextPage == 0 {
+					break
+				} else if opts.Page == endPage {
+					break
+				}
+
+				opts.Page = resp.NextPage
+			}
+		}(startPage, endPage)
+		startPage = endPage + 1
+	}
+	wg.Wait()
+	pi.DoneChan <- nil
+}