From d7181b1cf62ca21d2cb53e841d01dd6ec045b647 Mon Sep 17 00:00:00 2001 From: Ryan D McGuire Date: Tue, 16 Jan 2024 11:15:52 -0500 Subject: [PATCH] Start moving gitlab code to remote interface --- internal/cache/cache.go | 4 + internal/cache/cache_load.go | 23 ++++- internal/config/config.go | 7 ++ internal/remotes/gitlab/gitlab.go | 27 +++++ internal/remotes/gitlab/gitlab_api.go | 98 ++++++++++++++++++ internal/remotes/gitlab/gitlab_strean.go | 78 +++++++++++++++ internal/remotes/load/load.go | 4 + internal/remotes/remote/remote.go | 17 +++- internal/remotes/remotes.go | 87 +++++----------- internal/remotes/remotes_client.go | 60 ++++++++++++ internal/remotes/remotes_load.go | 120 ----------------------- 11 files changed, 339 insertions(+), 186 deletions(-) create mode 100644 internal/remotes/gitlab/gitlab.go create mode 100644 internal/remotes/gitlab/gitlab_api.go create mode 100644 internal/remotes/gitlab/gitlab_strean.go delete mode 100644 internal/remotes/remotes_load.go diff --git a/internal/cache/cache.go b/internal/cache/cache.go index c9687aa..6f6e072 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -10,6 +10,7 @@ import ( "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" "gopkg.in/yaml.v3" ) @@ -26,6 +27,7 @@ type Cache struct { log *pterm.Logger path string gitlabs *remotes.Clients + remotes *remote.Remotes } type CacheOpts struct { @@ -34,6 +36,7 @@ type CacheOpts struct { TTL time.Duration Logger *pterm.Logger Gitlabs *remotes.Clients + Remotes *remote.Remotes Config *config.Config } @@ -211,6 +214,7 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) { contentLock: &sync.Mutex{}, log: opts.Logger, gitlabs: gitlabs, + remotes: remote.NewRemotes(), path: opts.ProjectsPath, } diff --git a/internal/cache/cache_load.go b/internal/cache/cache_load.go index 149ba49..c863609 100644 --- a/internal/cache/cache_load.go +++ b/internal/cache/cache_load.go @@ -43,6 +43,27 @@ func (c *Cache) LoadGitlabs() { fmt.Println("") } +func (c *Cache) LoadRemote(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) { + defer wg.Done() + progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects) + + for { + select { + case p := <-progressInfo.ProgressChan: + pBar.Add(p.Projects) + case p := <-progressInfo.ProjectsChan: + c.AddProjects(p...) + case e := <-progressInfo.ErrorChan: + c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name)) + case <-client.Ctx.Done(): + c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err())) + return + case <-progressInfo.DoneChan: + return + } + } +} + func (c *Cache) LoadGitlab(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) { defer wg.Done() progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects) @@ -54,7 +75,7 @@ func (c *Cache) LoadGitlab(client *remotes.Client, wg *sync.WaitGroup, pBar *pte case p := <-progressInfo.ProjectsChan: c.AddProjects(p...) case e := <-progressInfo.ErrorChan: - c.log.Error("Fetch GitLab projects error", c.log.Args("error", e, "gitlab", client.Config.Name)) + c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name)) case <-client.Ctx.Done(): c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err())) return diff --git a/internal/config/config.go b/internal/config/config.go index 3e74658..9b15903 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,6 +8,7 @@ type Config struct { GitlabHost string `yaml:"gitlabHost,omitempty" json:"gitlabHost,omitempty"` GitlabToken string `yaml:"gitlabToken,omitempty" json:"gitlabToken,omitempty"` Gitlabs []GitlabConfig `yaml:"gitlabs" json:"gitlabs"` + Giteas []GiteaConfig `yaml:"giteas" json:"giteas"` LogLevel string `yaml:"logLevel" json:"logLevel" enum:"info,warn,debug,error"` ProjectPath string `yaml:"projectPath" json:"projectPath"` Cache cacheConfig `yaml:"cache" json:"cache"` @@ -17,6 +18,12 @@ type Config struct { Editor editorConfig `yaml:"editor" json:"editor"` } +type GiteaConfig struct { + Host string `yaml:"host" json:"host"` + Name string `yaml:"name" json:"name"` + Token string `yaml:"token" json:"token"` +} + type GitlabConfig struct { Host string `yaml:"host" json:"host"` Name string `yaml:"name" json:"name"` diff --git a/internal/remotes/gitlab/gitlab.go b/internal/remotes/gitlab/gitlab.go new file mode 100644 index 0000000..f7c2b9f --- /dev/null +++ b/internal/remotes/gitlab/gitlab.go @@ -0,0 +1,27 @@ +package gitlabremote + +import ( + "github.com/xanzy/go-gitlab" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" +) + +type GitlabRemote struct { + info *remote.RemoteInfo + api *gitlab.Client +} + +func (r *GitlabRemote) GetInfo() *remote.RemoteInfo { + return r.info +} + +func NewGitlabRemote(remoteInfo *remote.RemoteInfo) (*GitlabRemote, error) { + api, err := NewGitlabApi(remoteInfo) + if err != nil { + return nil, err + } + gl := &GitlabRemote{ + info: remoteInfo, + api: api, + } + return gl, nil +} diff --git a/internal/remotes/gitlab/gitlab_api.go b/internal/remotes/gitlab/gitlab_api.go new file mode 100644 index 0000000..ce140a4 --- /dev/null +++ b/internal/remotes/gitlab/gitlab_api.go @@ -0,0 +1,98 @@ +package gitlabremote + +import ( + "github.com/pterm/pterm" + "github.com/xanzy/go-gitlab" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" +) + +func NewGitlabApi(info *remote.RemoteInfo) (*gitlab.Client, error) { + client, err := gitlab.NewClient(info.Token, gitlab.WithBaseURL(info.Host)) + if err != nil { + return nil, err + } + return client, nil +} + +func (r *GitlabRemote) GetNumProjects(opts *remote.RemoteQueryOpts) int { + listOpts := *DefaultListOpts + listOpts.PerPage = 1 + listOpts.Simple = gitlab.Ptr[bool](true) + _, resp, err := r.api.Projects.ListProjects(&listOpts) + if err != nil { + pterm.Error.Printfln("Failed getting number of GitLab projects: %s", err) + return -1 + } + return resp.TotalItems +} + +// Returns a list of projects along with the next page and an error +// if there was an error +func (r *GitlabRemote) ListProjects(opts *gitlab.ListProjectsOptions) ( + []*projects.Project, *gitlab.Response, error) { + pList := make([]*projects.Project, 0) + projects, resp, err := r.api.Projects.ListProjects( + opts, + gitlab.WithContext(r.info.Ctx), + ) + if err == nil { + pList = append(pList, r.handleProjects(projects)...) + } + return pList, resp, err +} + +func (r *GitlabRemote) handleProjects(gitProjects []*gitlab.Project) []*projects.Project { + // Opportunity to perform any filtering or additional lookups + // on a per-project basis + pList := make([]*projects.Project, 0, len(gitProjects)) + for _, project := range gitProjects { + var owner string + if project.Owner != nil { + owner = project.Owner.Email + } + p := &projects.Project{ + ID: project.ID, + Description: project.Description, + SSHURLToRepo: project.SSHURLToRepo, + HTTPURLToRepo: project.HTTPURLToRepo, + WebURL: project.WebURL, + Name: project.Name, + NameWithNamespace: project.NameWithNamespace, + Path: project.Path, + PathWithNamespace: project.PathWithNamespace, + Remote: r.info.Host, + Owner: owner, + AvatarURL: project.AvatarURL, + LastActivityAt: *project.LastActivityAt, + Readme: project.ReadmeURL, + Languages: r.GetProjectLanguages(project), + } + pList = append(pList, p) + } + return pList +} + +// A nil return indicates an API error or GitLab doesn't know what +// language the project uses. +func (r *GitlabRemote) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages { + l, _, e := r.api.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(r.info.Ctx)) + if e != nil { + pterm.Error.Printfln("Failed requesting project languages: %s", e.Error()) + return nil + } + + var pLangs projects.ProjectLanguages + pLangs = make([]*projects.ProjectLanguage, len(*l)) + + var i int + for name, pcnt := range *l { + pLangs[i] = &projects.ProjectLanguage{ + Name: name, + Percentage: pcnt, + } + i++ + } + + return &pLangs +} diff --git a/internal/remotes/gitlab/gitlab_strean.go b/internal/remotes/gitlab/gitlab_strean.go new file mode 100644 index 0000000..587a311 --- /dev/null +++ b/internal/remotes/gitlab/gitlab_strean.go @@ -0,0 +1,78 @@ +package gitlabremote + +import ( + "sync" + + "github.com/xanzy/go-gitlab" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" +) + +// Will determine number of total projects, +// then based on projectsPerPage (request) and +// projectsPerGoroutine, will spin off goroutines +// with offsets +const ( + projectsPerPage = 20 + projectsPerGoroutine = 200 +) + +var DefaultListOpts = &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: projectsPerPage, + Page: 1, + }, + Archived: gitlab.Ptr[bool](false), +} + +func (r *GitlabRemote) StreamProjects(pi *load.ProgressInfo, opts *remote.RemoteQueryOpts) { + defer close(pi.ProgressChan) + defer close(pi.ProjectsChan) + + listOpts := *DefaultListOpts + listOpts.Owned = gitlab.Ptr[bool](opts.OwnerOnly) + + // Get total number of projects + numGoroutines := pi.NumProjects / projectsPerGoroutine + + wg := sync.WaitGroup{} + startPage := 1 + for i := 1; i <= numGoroutines+1; i++ { + wg.Add(1) + endPage := startPage + (projectsPerGoroutine / projectsPerPage) + go func(startPage int, endPage int) { + defer wg.Done() + opts := listOpts + opts.Page = startPage + for { + projects, resp, err := r.ListProjects(&opts) + + if err != nil { + pi.ErrorChan <- err + break + } + + pi.ProjectsChan <- projects + pi.ProgressChan <- load.Progress{ + Page: resp.CurrentPage, + Pages: resp.TotalPages, + Projects: len(projects), + TotalProjects: resp.TotalItems, + } + + // We're done when we have it all or our context is done + // or we've hit our total pages + if r.info.Ctx.Err() != nil || resp.NextPage == 0 { + break + } else if opts.Page == endPage { + break + } + + opts.Page = resp.NextPage + } + }(startPage, endPage) + startPage = endPage + 1 + } + wg.Wait() + pi.DoneChan <- nil +} diff --git a/internal/remotes/load/load.go b/internal/remotes/load/load.go index 5a9be7d..3c5f634 100644 --- a/internal/remotes/load/load.go +++ b/internal/remotes/load/load.go @@ -4,6 +4,10 @@ import ( "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" ) +// This package provides structs that serve +// as the interface between remotes, and any code +// that will call StreamProjects() on those remotes + type ProgressInfo struct { ProgressChan chan Progress ProjectsChan chan []*projects.Project diff --git a/internal/remotes/remote/remote.go b/internal/remotes/remote/remote.go index 1497469..2eb3576 100644 --- a/internal/remotes/remote/remote.go +++ b/internal/remotes/remote/remote.go @@ -1,13 +1,24 @@ package remote -import "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" +import ( + "context" + + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" +) + +type RemoteInfo struct { + Ctx context.Context + Host string + Name string + Token string +} // Any remote needs to be able to return // the number of projects the user has access to and also // stream all projects along with updates to channels // provided by *load.ProgressInfo type Remote interface { - Name() string + GetInfo() *RemoteInfo GetNumProjects(*RemoteQueryOpts) int - StreamProjects(*RemoteQueryOpts) *load.ProgressInfo + StreamProjects(*load.ProgressInfo, *RemoteQueryOpts) } diff --git a/internal/remotes/remotes.go b/internal/remotes/remotes.go index 666fd56..87e3d04 100644 --- a/internal/remotes/remotes.go +++ b/internal/remotes/remotes.go @@ -1,9 +1,9 @@ package remotes import ( - "github.com/pterm/pterm" - "github.com/xanzy/go-gitlab" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" ) // Will determine number of total projects, @@ -15,68 +15,31 @@ const ( projectsPerGoroutine = 200 ) -type User struct { - ID int - Username string - Email string - Name string - AvatarURL string +type Remotes []remote.Remote + +func (r *Remotes) AddRemote(remote remote.Remote) { + *r = append(*r, remote) } -func (c *Client) Api() *gitlab.Client { - return c.apiClient -} - -func (c *Client) GetTotalProjects(opts *gitlab.ListProjectsOptions) int { - reqOpts := *opts - reqOpts.ListOptions = gitlab.ListOptions{ - Page: 1, - PerPage: 1, - } - - var projects int - if _, r, e := c.apiClient.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil { - projects = r.TotalItems - } - - return projects -} - -// Returns a list of projects along with the next page and an error -// if there was an error -func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) ( - []*projects.Project, *gitlab.Response, error) { - pList := make([]*projects.Project, 0) - projects, resp, err := c.apiClient.Projects.ListProjects( - opts, - gitlab.WithContext(c.Ctx), - ) - if err == nil { - pList = append(pList, c.handleProjects(projects)...) - } - return pList, resp, err -} - -// A nil return indicates an API error or GitLab doesn't know what -// language the project uses. -func (c *Client) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages { - l, _, e := c.apiClient.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(c.Ctx)) - if e != nil { - pterm.Error.Printfln("Failed requesting project languages: %s", e.Error()) - return nil - } - - var pLangs projects.ProjectLanguages - pLangs = make([]*projects.ProjectLanguage, len(*l)) - - var i int - for name, pcnt := range *l { - pLangs[i] = &projects.ProjectLanguage{ - Name: name, - Percentage: pcnt, +// Launches project streamsers for all remotes in goroutines +// returns slice of load.ProgressInfo +func (r *Remotes) StreamRemotes(opts *remote.RemoteQueryOpts) []*load.ProgressInfo { + progressInfos := make([]*load.ProgressInfo, len(*r)) + for i, remoteInstance := range *r { + progressInfos[i] = &load.ProgressInfo{ + ProgressChan: make(chan load.Progress), + ProjectsChan: make(chan []*projects.Project), + ErrorChan: make(chan error), + DoneChan: make(chan interface{}), + NumProjects: remoteInstance.GetNumProjects(opts), } - i++ + go remoteInstance.StreamProjects(progressInfos[i], opts) } - - return &pLangs + return progressInfos +} + +func NewRemotes() *Remotes { + var remotes Remotes + remotes = make([]remote.Remote, 0) + return &remotes } diff --git a/internal/remotes/remotes_client.go b/internal/remotes/remotes_client.go index 41cdf37..e2f8f2a 100644 --- a/internal/remotes/remotes_client.go +++ b/internal/remotes/remotes_client.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" + "github.com/pterm/pterm" "github.com/xanzy/go-gitlab" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" ) type Client struct { @@ -81,3 +83,61 @@ func NewGitlabClient(opts *ClientOpts) (*Client, error) { } return gitlabClient, nil } + +func (c *Client) Api() *gitlab.Client { + return c.apiClient +} + +func (c *Client) GetTotalProjects(opts *gitlab.ListProjectsOptions) int { + reqOpts := *opts + reqOpts.ListOptions = gitlab.ListOptions{ + Page: 1, + PerPage: 1, + } + + var projects int + if _, r, e := c.apiClient.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil { + projects = r.TotalItems + } + + return projects +} + +// Returns a list of projects along with the next page and an error +// if there was an error +func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) ( + []*projects.Project, *gitlab.Response, error) { + pList := make([]*projects.Project, 0) + projects, resp, err := c.apiClient.Projects.ListProjects( + opts, + gitlab.WithContext(c.Ctx), + ) + if err == nil { + pList = append(pList, c.handleProjects(projects)...) + } + return pList, resp, err +} + +// A nil return indicates an API error or GitLab doesn't know what +// language the project uses. +func (c *Client) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages { + l, _, e := c.apiClient.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(c.Ctx)) + if e != nil { + pterm.Error.Printfln("Failed requesting project languages: %s", e.Error()) + return nil + } + + var pLangs projects.ProjectLanguages + pLangs = make([]*projects.ProjectLanguage, len(*l)) + + var i int + for name, pcnt := range *l { + pLangs[i] = &projects.ProjectLanguage{ + Name: name, + Percentage: pcnt, + } + i++ + } + + return &pLangs +} diff --git a/internal/remotes/remotes_load.go b/internal/remotes/remotes_load.go deleted file mode 100644 index 6c08155..0000000 --- a/internal/remotes/remotes_load.go +++ /dev/null @@ -1,120 +0,0 @@ -package remotes - -import ( - "fmt" - "sync" - - "github.com/xanzy/go-gitlab" - "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" - "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" -) - -var DefaultListOpts = &gitlab.ListProjectsOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: projectsPerPage, - Page: 1, - }, - Archived: gitlab.Ptr[bool](false), -} - -// Given there may be thousands of projects, this will return -// channels that stream progress info and then finally the full -// list of projects on separate channels. If ownerOnly=true, only -// projects for which you are an owner will be loaded -func (c *Client) StreamProjects(ownerOnly bool, numProjects int) *load.ProgressInfo { - fmt.Println(numProjects) - pi := &load.ProgressInfo{ - ProgressChan: make(chan load.Progress), - ProjectsChan: make(chan []*projects.Project), - ErrorChan: make(chan error), - DoneChan: make(chan interface{}), - NumProjects: numProjects, - } - - go c.streamProjects(pi, ownerOnly) - - return pi -} - -func (c *Client) streamProjects(pi *load.ProgressInfo, ownerOnly bool) { - defer close(pi.ProgressChan) - defer close(pi.ProjectsChan) - - listOpts := *DefaultListOpts - listOpts.Owned = gitlab.Ptr[bool](ownerOnly) - - // Get total number of projects - numGoroutines := pi.NumProjects / projectsPerGoroutine - - wg := sync.WaitGroup{} - startPage := 1 - for i := 1; i <= numGoroutines+1; i++ { - wg.Add(1) - endPage := startPage + (projectsPerGoroutine / projectsPerPage) - go func(startPage int, endPage int) { - defer wg.Done() - opts := listOpts - opts.Page = startPage - for { - projects, resp, err := c.ListProjects(&opts) - - if err != nil { - pi.ErrorChan <- err - break - } - - pi.ProjectsChan <- projects - pi.ProgressChan <- load.Progress{ - Page: resp.CurrentPage, - Pages: resp.TotalPages, - Projects: len(projects), - TotalProjects: resp.TotalItems, - } - - // We're done when we have it all or our context is done - // or we've hit our total pages - if c.Ctx.Err() != nil || resp.NextPage == 0 { - break - } else if opts.Page == endPage { - break - } - - opts.Page = resp.NextPage - } - }(startPage, endPage) - startPage = endPage + 1 - } - wg.Wait() - pi.DoneChan <- nil -} - -func (c *Client) handleProjects(gitProjects []*gitlab.Project) []*projects.Project { - // Opportunity to perform any filtering or additional lookups - // on a per-project basis - pList := make([]*projects.Project, 0, len(gitProjects)) - for _, project := range gitProjects { - var owner string - if project.Owner != nil { - owner = project.Owner.Email - } - p := &projects.Project{ - ID: project.ID, - Description: project.Description, - SSHURLToRepo: project.SSHURLToRepo, - HTTPURLToRepo: project.HTTPURLToRepo, - WebURL: project.WebURL, - Name: project.Name, - NameWithNamespace: project.NameWithNamespace, - Path: project.Path, - PathWithNamespace: project.PathWithNamespace, - Remote: c.Config.Host, - Owner: owner, - AvatarURL: project.AvatarURL, - LastActivityAt: *project.LastActivityAt, - Readme: project.ReadmeURL, - Languages: c.GetProjectLanguages(project), - } - pList = append(pList, p) - } - return pList -}