Start moving gitlab code to remote interface

internal/cache/cache.go

@@ -10,6 +10,7 @@ import (
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
 	"gopkg.in/yaml.v3"
 )
 
@@ -26,6 +27,7 @@ type Cache struct {
 	log          *pterm.Logger
 	path         string
 	gitlabs      *remotes.Clients
+	remotes      *remote.Remotes
 }
 
 type CacheOpts struct {
@@ -34,6 +36,7 @@ type CacheOpts struct {
 	TTL          time.Duration
 	Logger       *pterm.Logger
 	Gitlabs      *remotes.Clients
+	Remotes      *remote.Remotes
 	Config       *config.Config
 }
 
@@ -211,6 +214,7 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) {
 		contentLock: &sync.Mutex{},
 		log:         opts.Logger,
 		gitlabs:     gitlabs,
+		remotes:     remote.NewRemotes(),
 		path:        opts.ProjectsPath,
 	}
 

internal/cache/cache_load.go

@@ -43,6 +43,27 @@ func (c *Cache) LoadGitlabs() {
 	fmt.Println("")
 }
 
+func (c *Cache) LoadRemote(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) {
+	defer wg.Done()
+	progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects)
+
+	for {
+		select {
+		case p := <-progressInfo.ProgressChan:
+			pBar.Add(p.Projects)
+		case p := <-progressInfo.ProjectsChan:
+			c.AddProjects(p...)
+		case e := <-progressInfo.ErrorChan:
+			c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name))
+		case <-client.Ctx.Done():
+			c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err()))
+			return
+		case <-progressInfo.DoneChan:
+			return
+		}
+	}
+}
+
 func (c *Cache) LoadGitlab(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) {
 	defer wg.Done()
 	progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects)
@@ -54,7 +75,7 @@ func (c *Cache) LoadGitlab(client *remotes.Client, wg *sync.WaitGroup, pBar *pte
 		case p := <-progressInfo.ProjectsChan:
 			c.AddProjects(p...)
 		case e := <-progressInfo.ErrorChan:
-			c.log.Error("Fetch GitLab projects error", c.log.Args("error", e, "gitlab", client.Config.Name))
+			c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name))
 		case <-client.Ctx.Done():
 			c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err()))
 			return

internal/config/config.go

@@ -8,6 +8,7 @@ type Config struct {
 	GitlabHost  string         `yaml:"gitlabHost,omitempty" json:"gitlabHost,omitempty"`
 	GitlabToken string         `yaml:"gitlabToken,omitempty" json:"gitlabToken,omitempty"`
 	Gitlabs     []GitlabConfig `yaml:"gitlabs" json:"gitlabs"`
+	Giteas      []GiteaConfig  `yaml:"giteas" json:"giteas"`
 	LogLevel    string         `yaml:"logLevel" json:"logLevel" enum:"info,warn,debug,error"`
 	ProjectPath string         `yaml:"projectPath" json:"projectPath"`
 	Cache       cacheConfig    `yaml:"cache" json:"cache"`
@@ -17,6 +18,12 @@ type Config struct {
 	Editor editorConfig `yaml:"editor" json:"editor"`
 }
 
+type GiteaConfig struct {
+	Host  string `yaml:"host" json:"host"`
+	Name  string `yaml:"name" json:"name"`
+	Token string `yaml:"token" json:"token"`
+}
+
 type GitlabConfig struct {
 	Host  string `yaml:"host" json:"host"`
 	Name  string `yaml:"name" json:"name"`

internal/remotes/gitlab/gitlab.go

@@ -0,0 +1,27 @@
+package gitlabremote
+
+import (
+	"github.com/xanzy/go-gitlab"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
+)
+
+type GitlabRemote struct {
+	info *remote.RemoteInfo
+	api  *gitlab.Client
+}
+
+func (r *GitlabRemote) GetInfo() *remote.RemoteInfo {
+	return r.info
+}
+
+func NewGitlabRemote(remoteInfo *remote.RemoteInfo) (*GitlabRemote, error) {
+	api, err := NewGitlabApi(remoteInfo)
+	if err != nil {
+		return nil, err
+	}
+	gl := &GitlabRemote{
+		info: remoteInfo,
+		api:  api,
+	}
+	return gl, nil
+}

internal/remotes/gitlab/gitlab_api.go

@@ -0,0 +1,98 @@
+package gitlabremote
+
+import (
+	"github.com/pterm/pterm"
+	"github.com/xanzy/go-gitlab"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
+)
+
+func NewGitlabApi(info *remote.RemoteInfo) (*gitlab.Client, error) {
+	client, err := gitlab.NewClient(info.Token, gitlab.WithBaseURL(info.Host))
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+func (r *GitlabRemote) GetNumProjects(opts *remote.RemoteQueryOpts) int {
+	listOpts := *DefaultListOpts
+	listOpts.PerPage = 1
+	listOpts.Simple = gitlab.Ptr[bool](true)
+	_, resp, err := r.api.Projects.ListProjects(&listOpts)
+	if err != nil {
+		pterm.Error.Printfln("Failed getting number of GitLab projects: %s", err)
+		return -1
+	}
+	return resp.TotalItems
+}
+
+// Returns a list of projects along with the next page and an error
+// if there was an error
+func (r *GitlabRemote) ListProjects(opts *gitlab.ListProjectsOptions) (
+	[]*projects.Project, *gitlab.Response, error) {
+	pList := make([]*projects.Project, 0)
+	projects, resp, err := r.api.Projects.ListProjects(
+		opts,
+		gitlab.WithContext(r.info.Ctx),
+	)
+	if err == nil {
+		pList = append(pList, r.handleProjects(projects)...)
+	}
+	return pList, resp, err
+}
+
+func (r *GitlabRemote) handleProjects(gitProjects []*gitlab.Project) []*projects.Project {
+	// Opportunity to perform any filtering or additional lookups
+	// on a per-project basis
+	pList := make([]*projects.Project, 0, len(gitProjects))
+	for _, project := range gitProjects {
+		var owner string
+		if project.Owner != nil {
+			owner = project.Owner.Email
+		}
+		p := &projects.Project{
+			ID:                project.ID,
+			Description:       project.Description,
+			SSHURLToRepo:      project.SSHURLToRepo,
+			HTTPURLToRepo:     project.HTTPURLToRepo,
+			WebURL:            project.WebURL,
+			Name:              project.Name,
+			NameWithNamespace: project.NameWithNamespace,
+			Path:              project.Path,
+			PathWithNamespace: project.PathWithNamespace,
+			Remote:            r.info.Host,
+			Owner:             owner,
+			AvatarURL:         project.AvatarURL,
+			LastActivityAt:    *project.LastActivityAt,
+			Readme:            project.ReadmeURL,
+			Languages:         r.GetProjectLanguages(project),
+		}
+		pList = append(pList, p)
+	}
+	return pList
+}
+
+// A nil return indicates an API error or GitLab doesn't know what
+// language the project uses.
+func (r *GitlabRemote) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages {
+	l, _, e := r.api.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(r.info.Ctx))
+	if e != nil {
+		pterm.Error.Printfln("Failed requesting project languages: %s", e.Error())
+		return nil
+	}
+
+	var pLangs projects.ProjectLanguages
+	pLangs = make([]*projects.ProjectLanguage, len(*l))
+
+	var i int
+	for name, pcnt := range *l {
+		pLangs[i] = &projects.ProjectLanguage{
+			Name:       name,
+			Percentage: pcnt,
+		}
+		i++
+	}
+
+	return &pLangs
+}

internal/remotes/gitlab/gitlab_strean.go

@@ -0,0 +1,78 @@
+package gitlabremote
+
+import (
+	"sync"
+
+	"github.com/xanzy/go-gitlab"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
+)
+
+// Will determine number of total projects,
+// then based on projectsPerPage (request) and
+// projectsPerGoroutine, will spin off goroutines
+// with offsets
+const (
+	projectsPerPage      = 20
+	projectsPerGoroutine = 200
+)
+
+var DefaultListOpts = &gitlab.ListProjectsOptions{
+	ListOptions: gitlab.ListOptions{
+		PerPage: projectsPerPage,
+		Page:    1,
+	},
+	Archived: gitlab.Ptr[bool](false),
+}
+
+func (r *GitlabRemote) StreamProjects(pi *load.ProgressInfo, opts *remote.RemoteQueryOpts) {
+	defer close(pi.ProgressChan)
+	defer close(pi.ProjectsChan)
+
+	listOpts := *DefaultListOpts
+	listOpts.Owned = gitlab.Ptr[bool](opts.OwnerOnly)
+
+	// Get total number of projects
+	numGoroutines := pi.NumProjects / projectsPerGoroutine
+
+	wg := sync.WaitGroup{}
+	startPage := 1
+	for i := 1; i <= numGoroutines+1; i++ {
+		wg.Add(1)
+		endPage := startPage + (projectsPerGoroutine / projectsPerPage)
+		go func(startPage int, endPage int) {
+			defer wg.Done()
+			opts := listOpts
+			opts.Page = startPage
+			for {
+				projects, resp, err := r.ListProjects(&opts)
+
+				if err != nil {
+					pi.ErrorChan <- err
+					break
+				}
+
+				pi.ProjectsChan <- projects
+				pi.ProgressChan <- load.Progress{
+					Page:          resp.CurrentPage,
+					Pages:         resp.TotalPages,
+					Projects:      len(projects),
+					TotalProjects: resp.TotalItems,
+				}
+
+				// We're done when we have it all or our context is done
+				// or we've hit our total pages
+				if r.info.Ctx.Err() != nil || resp.NextPage == 0 {
+					break
+				} else if opts.Page == endPage {
+					break
+				}
+
+				opts.Page = resp.NextPage
+			}
+		}(startPage, endPage)
+		startPage = endPage + 1
+	}
+	wg.Wait()
+	pi.DoneChan <- nil
+}

internal/remotes/load/load.go

@@ -4,6 +4,10 @@ import (
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
 )
 
+// This package provides structs that serve
+// as the interface between remotes, and any code
+// that will call StreamProjects() on those remotes
+
 type ProgressInfo struct {
 	ProgressChan chan Progress
 	ProjectsChan chan []*projects.Project

internal/remotes/remote/remote.go

@@ -1,13 +1,24 @@
 package remote
 
-import "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
+import (
+	"context"
+
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
+)
+
+type RemoteInfo struct {
+	Ctx   context.Context
+	Host  string
+	Name  string
+	Token string
+}
 
 // Any remote needs to be able to return
 // the number of projects the user has access to and also
 // stream all projects along with updates to channels
 // provided by *load.ProgressInfo
 type Remote interface {
-	Name() string
+	GetInfo() *RemoteInfo
 	GetNumProjects(*RemoteQueryOpts) int
-	StreamProjects(*RemoteQueryOpts) *load.ProgressInfo
+	StreamProjects(*load.ProgressInfo, *RemoteQueryOpts)
 }

internal/remotes/remotes.go

@@ -1,9 +1,9 @@
 package remotes
 
 import (
-	"github.com/pterm/pterm"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
-	"github.com/xanzy/go-gitlab"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
 )
 
 // Will determine number of total projects,
@@ -15,68 +15,31 @@ const (
 	projectsPerGoroutine = 200
 )
 
-type User struct {
+type Remotes []remote.Remote
-	ID        int
+
-	Username  string
+func (r *Remotes) AddRemote(remote remote.Remote) {
-	Email     string
+	*r = append(*r, remote)
-	Name      string
-	AvatarURL string
 }
 
-func (c *Client) Api() *gitlab.Client {
+// Launches project streamsers for all remotes in goroutines
-	return c.apiClient
+// returns slice of load.ProgressInfo
-}
+func (r *Remotes) StreamRemotes(opts *remote.RemoteQueryOpts) []*load.ProgressInfo {
-
+	progressInfos := make([]*load.ProgressInfo, len(*r))
-func (c *Client) GetTotalProjects(opts *gitlab.ListProjectsOptions) int {
+	for i, remoteInstance := range *r {
-	reqOpts := *opts
+		progressInfos[i] = &load.ProgressInfo{
-	reqOpts.ListOptions = gitlab.ListOptions{
+			ProgressChan: make(chan load.Progress),
-		Page:    1,
+			ProjectsChan: make(chan []*projects.Project),
-		PerPage: 1,
+			ErrorChan:    make(chan error),
-	}
+			DoneChan:     make(chan interface{}),
-
+			NumProjects:  remoteInstance.GetNumProjects(opts),
-	var projects int
-	if _, r, e := c.apiClient.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil {
-		projects = r.TotalItems
-	}
-
-	return projects
-}
-
-// Returns a list of projects along with the next page and an error
-// if there was an error
-func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) (
-	[]*projects.Project, *gitlab.Response, error) {
-	pList := make([]*projects.Project, 0)
-	projects, resp, err := c.apiClient.Projects.ListProjects(
-		opts,
-		gitlab.WithContext(c.Ctx),
-	)
-	if err == nil {
-		pList = append(pList, c.handleProjects(projects)...)
-	}
-	return pList, resp, err
-}
-
-// A nil return indicates an API error or GitLab doesn't know what
-// language the project uses.
-func (c *Client) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages {
-	l, _, e := c.apiClient.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(c.Ctx))
-	if e != nil {
-		pterm.Error.Printfln("Failed requesting project languages: %s", e.Error())
-		return nil
-	}
-
-	var pLangs projects.ProjectLanguages
-	pLangs = make([]*projects.ProjectLanguage, len(*l))
-
-	var i int
-	for name, pcnt := range *l {
-		pLangs[i] = &projects.ProjectLanguage{
-			Name:       name,
-			Percentage: pcnt,
 		}
-		i++
+		go remoteInstance.StreamProjects(progressInfos[i], opts)
 	}
-
+	return progressInfos
-	return &pLangs
+}
+
+func NewRemotes() *Remotes {
+	var remotes Remotes
+	remotes = make([]remote.Remote, 0)
+	return &remotes
 }

internal/remotes/remotes_client.go

@@ -5,8 +5,10 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/pterm/pterm"
 	"github.com/xanzy/go-gitlab"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
 )
 
 type Client struct {
@@ -81,3 +83,61 @@ func NewGitlabClient(opts *ClientOpts) (*Client, error) {
 	}
 	return gitlabClient, nil
 }
+
+func (c *Client) Api() *gitlab.Client {
+	return c.apiClient
+}
+
+func (c *Client) GetTotalProjects(opts *gitlab.ListProjectsOptions) int {
+	reqOpts := *opts
+	reqOpts.ListOptions = gitlab.ListOptions{
+		Page:    1,
+		PerPage: 1,
+	}
+
+	var projects int
+	if _, r, e := c.apiClient.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil {
+		projects = r.TotalItems
+	}
+
+	return projects
+}
+
+// Returns a list of projects along with the next page and an error
+// if there was an error
+func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) (
+	[]*projects.Project, *gitlab.Response, error) {
+	pList := make([]*projects.Project, 0)
+	projects, resp, err := c.apiClient.Projects.ListProjects(
+		opts,
+		gitlab.WithContext(c.Ctx),
+	)
+	if err == nil {
+		pList = append(pList, c.handleProjects(projects)...)
+	}
+	return pList, resp, err
+}
+
+// A nil return indicates an API error or GitLab doesn't know what
+// language the project uses.
+func (c *Client) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages {
+	l, _, e := c.apiClient.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(c.Ctx))
+	if e != nil {
+		pterm.Error.Printfln("Failed requesting project languages: %s", e.Error())
+		return nil
+	}
+
+	var pLangs projects.ProjectLanguages
+	pLangs = make([]*projects.ProjectLanguage, len(*l))
+
+	var i int
+	for name, pcnt := range *l {
+		pLangs[i] = &projects.ProjectLanguage{
+			Name:       name,
+			Percentage: pcnt,
+		}
+		i++
+	}
+
+	return &pLangs
+}

internal/remotes/remotes_load.go

@@ -1,120 +0,0 @@
-package remotes
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/xanzy/go-gitlab"
-	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
-	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
-)
-
-var DefaultListOpts = &gitlab.ListProjectsOptions{
-	ListOptions: gitlab.ListOptions{
-		PerPage: projectsPerPage,
-		Page:    1,
-	},
-	Archived: gitlab.Ptr[bool](false),
-}
-
-// Given there may be thousands of projects, this will return
-// channels that stream progress info and then finally the full
-// list of projects on separate channels. If ownerOnly=true, only
-// projects for which you are an owner will be loaded
-func (c *Client) StreamProjects(ownerOnly bool, numProjects int) *load.ProgressInfo {
-	fmt.Println(numProjects)
-	pi := &load.ProgressInfo{
-		ProgressChan: make(chan load.Progress),
-		ProjectsChan: make(chan []*projects.Project),
-		ErrorChan:    make(chan error),
-		DoneChan:     make(chan interface{}),
-		NumProjects:  numProjects,
-	}
-
-	go c.streamProjects(pi, ownerOnly)
-
-	return pi
-}
-
-func (c *Client) streamProjects(pi *load.ProgressInfo, ownerOnly bool) {
-	defer close(pi.ProgressChan)
-	defer close(pi.ProjectsChan)
-
-	listOpts := *DefaultListOpts
-	listOpts.Owned = gitlab.Ptr[bool](ownerOnly)
-
-	// Get total number of projects
-	numGoroutines := pi.NumProjects / projectsPerGoroutine
-
-	wg := sync.WaitGroup{}
-	startPage := 1
-	for i := 1; i <= numGoroutines+1; i++ {
-		wg.Add(1)
-		endPage := startPage + (projectsPerGoroutine / projectsPerPage)
-		go func(startPage int, endPage int) {
-			defer wg.Done()
-			opts := listOpts
-			opts.Page = startPage
-			for {
-				projects, resp, err := c.ListProjects(&opts)
-
-				if err != nil {
-					pi.ErrorChan <- err
-					break
-				}
-
-				pi.ProjectsChan <- projects
-				pi.ProgressChan <- load.Progress{
-					Page:          resp.CurrentPage,
-					Pages:         resp.TotalPages,
-					Projects:      len(projects),
-					TotalProjects: resp.TotalItems,
-				}
-
-				// We're done when we have it all or our context is done
-				// or we've hit our total pages
-				if c.Ctx.Err() != nil || resp.NextPage == 0 {
-					break
-				} else if opts.Page == endPage {
-					break
-				}
-
-				opts.Page = resp.NextPage
-			}
-		}(startPage, endPage)
-		startPage = endPage + 1
-	}
-	wg.Wait()
-	pi.DoneChan <- nil
-}
-
-func (c *Client) handleProjects(gitProjects []*gitlab.Project) []*projects.Project {
-	// Opportunity to perform any filtering or additional lookups
-	// on a per-project basis
-	pList := make([]*projects.Project, 0, len(gitProjects))
-	for _, project := range gitProjects {
-		var owner string
-		if project.Owner != nil {
-			owner = project.Owner.Email
-		}
-		p := &projects.Project{
-			ID:                project.ID,
-			Description:       project.Description,
-			SSHURLToRepo:      project.SSHURLToRepo,
-			HTTPURLToRepo:     project.HTTPURLToRepo,
-			WebURL:            project.WebURL,
-			Name:              project.Name,
-			NameWithNamespace: project.NameWithNamespace,
-			Path:              project.Path,
-			PathWithNamespace: project.PathWithNamespace,
-			Remote:            c.Config.Host,
-			Owner:             owner,
-			AvatarURL:         project.AvatarURL,
-			LastActivityAt:    *project.LastActivityAt,
-			Readme:            project.ReadmeURL,
-			Languages:         c.GetProjectLanguages(project),
-		}
-		pList = append(pList, p)
-	}
-	return pList
-}