package remotes import ( "fmt" "sync" "github.com/xanzy/go-gitlab" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" ) var DefaultListOpts = &gitlab.ListProjectsOptions{ ListOptions: gitlab.ListOptions{ PerPage: projectsPerPage, Page: 1, }, Archived: gitlab.Ptr[bool](false), } // Given there may be thousands of projects, this will return // channels that stream progress info and then finally the full // list of projects on separate channels. If ownerOnly=true, only // projects for which you are an owner will be loaded func (c *Client) StreamProjects(ownerOnly bool, numProjects int) *load.ProgressInfo { fmt.Println(numProjects) pi := &load.ProgressInfo{ ProgressChan: make(chan load.Progress), ProjectsChan: make(chan []*projects.Project), ErrorChan: make(chan error), DoneChan: make(chan interface{}), NumProjects: numProjects, } go c.streamProjects(pi, ownerOnly) return pi } func (c *Client) streamProjects(pi *load.ProgressInfo, ownerOnly bool) { defer close(pi.ProgressChan) defer close(pi.ProjectsChan) listOpts := *DefaultListOpts listOpts.Owned = gitlab.Ptr[bool](ownerOnly) // Get total number of projects numGoroutines := pi.NumProjects / projectsPerGoroutine wg := sync.WaitGroup{} startPage := 1 for i := 1; i <= numGoroutines+1; i++ { wg.Add(1) endPage := startPage + (projectsPerGoroutine / projectsPerPage) go func(startPage int, endPage int) { defer wg.Done() opts := listOpts opts.Page = startPage for { projects, resp, err := c.ListProjects(&opts) if err != nil { pi.ErrorChan <- err break } pi.ProjectsChan <- projects pi.ProgressChan <- load.Progress{ Page: resp.CurrentPage, Pages: resp.TotalPages, Projects: len(projects), TotalProjects: resp.TotalItems, } // We're done when we have it all or our context is done // or we've hit our total pages if c.Ctx.Err() != nil || resp.NextPage == 0 { break } else if opts.Page == endPage { break } opts.Page = resp.NextPage } }(startPage, endPage) startPage = endPage + 1 } wg.Wait() pi.DoneChan <- nil } func (c *Client) handleProjects(gitProjects []*gitlab.Project) []*projects.Project { // Opportunity to perform any filtering or additional lookups // on a per-project basis pList := make([]*projects.Project, 0, len(gitProjects)) for _, project := range gitProjects { var owner string if project.Owner != nil { owner = project.Owner.Email } p := &projects.Project{ ID: project.ID, Description: project.Description, SSHURLToRepo: project.SSHURLToRepo, HTTPURLToRepo: project.HTTPURLToRepo, WebURL: project.WebURL, Name: project.Name, NameWithNamespace: project.NameWithNamespace, Path: project.Path, PathWithNamespace: project.PathWithNamespace, Remote: c.Config.Host, Owner: owner, AvatarURL: project.AvatarURL, LastActivityAt: *project.LastActivityAt, Readme: project.ReadmeURL, Languages: c.GetProjectLanguages(project), } pList = append(pList, p) } return pList }