package remotes import ( "sync" "github.com/xanzy/go-gitlab" ) type ProgressInfo struct { ProgressChan chan Progress ProjectsChan chan []*Project ErrorChan chan error DoneChan chan interface{} NumProjects int } type Progress struct { Page int Pages int Projects int TotalProjects int } var DefaultListOpts = &gitlab.ListProjectsOptions{ ListOptions: gitlab.ListOptions{ PerPage: projectsPerPage, Page: 1, }, Archived: gitlab.Ptr[bool](false), } // Given there may be thousands of projects, this will return // channels that stream progress info and then finally the full // list of projects on separate channels. If ownerOnly=true, only // projects for which you are an owner will be loaded func (c *Client) StreamProjects(ownerOnly bool, projects int) *ProgressInfo { pi := &ProgressInfo{ ProgressChan: make(chan Progress), ProjectsChan: make(chan []*Project), ErrorChan: make(chan error), DoneChan: make(chan interface{}), NumProjects: projects, } go c.streamProjects(pi, ownerOnly) return pi } func (c *Client) streamProjects(pi *ProgressInfo, ownerOnly bool) { defer close(pi.ProgressChan) defer close(pi.ProjectsChan) listOpts := *DefaultListOpts listOpts.Owned = gitlab.Ptr[bool](ownerOnly) // Get total number of projects numGoroutines := pi.NumProjects / projectsPerGoroutine wg := sync.WaitGroup{} startPage := 1 for i := 1; i <= numGoroutines+1; i++ { wg.Add(1) endPage := startPage + (projectsPerGoroutine / projectsPerPage) go func(startPage int, endPage int) { defer wg.Done() opts := listOpts opts.Page = startPage for { projects, resp, err := c.ListProjects(&opts) if err != nil { pi.ErrorChan <- err break } pi.ProjectsChan <- projects pi.ProgressChan <- Progress{ Page: resp.CurrentPage, Pages: resp.TotalPages, Projects: len(projects), TotalProjects: resp.TotalItems, } // We're done when we have it all or our context is done // or we've hit our total pages if c.Ctx.Err() != nil || resp.NextPage == 0 { break } else if opts.Page == endPage { break } opts.Page = resp.NextPage } }(startPage, endPage) startPage = endPage + 1 } wg.Wait() pi.DoneChan <- nil } func (c *Client) handleProjects(projects []*gitlab.Project) []*Project { // Opportunity to perform any filtering or additional lookups // on a per-project basis pList := make([]*Project, 0, len(projects)) for _, project := range projects { var owner string if project.Owner != nil { owner = project.Owner.Email } p := &Project{ ID: project.ID, Description: project.Description, SSHURLToRepo: project.SSHURLToRepo, HTTPURLToRepo: project.HTTPURLToRepo, WebURL: project.WebURL, Name: project.Name, NameWithNamespace: project.NameWithNamespace, Path: project.Path, PathWithNamespace: project.PathWithNamespace, Remote: c.Config.Host, Owner: owner, AvatarURL: project.AvatarURL, LastActivityAt: *project.LastActivityAt, Readme: project.ReadmeURL, Languages: c.GetProjectLanguages(project), } pList = append(pList, p) } return pList }