git-project-manager/internal/remotes/gitlab/gitlab_strean.go

package gitlabremote

import (
	"sync"

	"github.com/xanzy/go-gitlab"
	"gitea.libretechconsulting.com/rmcguire/git-project-manager/internal/remotes/load"
	"gitea.libretechconsulting.com/rmcguire/git-project-manager/internal/remotes/remote"
)

// Will determine number of total projects,
// then based on projectsPerPage (request) and
// projectsPerGoroutine, will spin off goroutines
// with offsets
const (
	projectsPerPage      = 20
	projectsPerGoroutine = 200
)

func (r *GitlabRemote) StreamProjects(pi *load.ProgressInfo, opts *remote.RemoteQueryOpts) {
	defer close(pi.ProgressChan)
	defer close(pi.ProjectsChan)

	listOpts := *DefaultListOpts
	listOpts.Owned = gitlab.Ptr[bool](opts.OwnerOnly)

	// Get total number of projects
	numGoroutines := pi.NumProjects / projectsPerGoroutine

	wg := sync.WaitGroup{}
	startPage := 1
	for i := 1; i <= numGoroutines+1; i++ {
		wg.Add(1)
		endPage := startPage + (projectsPerGoroutine / projectsPerPage)
		go func(startPage int, endPage int) {
			defer wg.Done()
			opts := listOpts
			opts.Page = startPage
			for {
				projects, resp, err := r.ListProjects(&opts)

				if err != nil {
					pi.ErrorChan <- err
					break
				}

				pi.ProjectsChan <- projects
				pi.ProgressChan <- load.Progress{
					Page:          resp.CurrentPage,
					Pages:         resp.TotalPages,
					Projects:      len(projects),
					TotalProjects: resp.TotalItems,
				}

				// We're done when we have it all or our context is done
				// or we've hit our total pages
				if r.info.Context().Err() != nil || resp.NextPage == 0 {
					break
				} else if opts.Page == endPage {
					break
				}

				opts.Page = resp.NextPage
			}
		}(startPage, endPage)
		startPage = endPage + 1
	}
	wg.Wait()
	pi.DoneChan <- nil
}