Migrate to remotes interface

cmd/config_generate.go

@@ -101,6 +101,17 @@ func promptConfigSettings(c *config.Config) *config.Config {
 		gitlabConfig.Token = token
 	}
 
+	if proto, err := pterm.DefaultInteractiveSelect.
+		WithOptions([]string{string(config.CloneProtoHTTP), string(config.CloneProtoSSH)}).
+		WithDefaultText("Git Clone Protocol").
+		Show(); err == nil {
+		if proto == "ssh" {
+			gitlabConfig.CloneProto = config.CloneProtoSSH
+		} else {
+			gitlabConfig.CloneProto = config.CloneProtoHTTP
+		}
+	}
+
 	if pPath, err := pterm.DefaultInteractiveTextInput.
 		WithDefaultValue(c.ProjectPath).
 		WithDefaultText("Enter path for projects and cache").

cmd/util_init.go

@@ -10,6 +10,8 @@ import (
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/cache"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes"
+	gitlabremote "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/gitlab"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
 	"golang.org/x/sys/unix"
 )
 
@@ -19,52 +21,19 @@ import (
 // func from their PersistentPreRun commands
 
 func initProjectCache(cmd *cobra.Command, args []string) {
+	var err error
 	plog.Debug("Running pre-run for cacheCmd")
 	conf.Cache.File = conf.ProjectPath + "/.cache.yaml"
 
-	// Backwards-compatible support for singular instance
-	opts := make([]*remotes.ClientOpts, 0)
-
-	if conf.GitlabHost != "" {
-		opts = append(opts, &remotes.ClientOpts{
-			Ctx:   cmd.Context(),
-			Host:  conf.GitlabHost,  // deprecated, switch to gitlabs
-			Token: conf.GitlabToken, // deprecated, switch to gitlabs
-			Name:  conf.GitlabHost,  // not originally supported, use the new gitlabs field
-		})
-	}
-
-	// If defined, load additional instances
-	for _, g := range conf.Gitlabs {
-		opts = append(opts, &remotes.ClientOpts{
-			Ctx:   cmd.Context(),
-			Name:  g.Name,
-			Host:  g.Host,
-			Token: g.Token,
-		})
-	}
-
-	// We need at least one GitLab
-	if len(opts) < 1 {
-		plog.Error("At least one GitLab must be configured. Add to .gitlabs in your config file")
-		os.Exit(1)
-	}
-
-	// Load all gitlab configs into clients
-	var gitlabs *remotes.Clients
-	var err error
-	gitlabs, err = remotes.NewGitlabClients(opts)
-	if err != nil {
-		plog.Error("Failed to create GitLab clients", plog.Args("error", err))
-		os.Exit(1)
-	}
+	gitRemotes := remotes.NewRemotes()
+	gitRemotes.AddRemotes(getGitLabRemotes(cmd)...)
 
 	cacheOpts := &cache.CacheOpts{
 		ProjectsPath: conf.ProjectPath,
 		Path:         conf.Cache.File,
 		TTL:          conf.Cache.Ttl,
 		Logger:       plog,
-		Gitlabs:      gitlabs,
+		Remotes:      gitRemotes,
 		Config:       &conf,
 	}
 	if projectCache, err = cache.NewProjectCache(cacheOpts); err != nil {
@@ -77,7 +46,39 @@ func initProjectCache(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	plog.Debug("Gitlab Clients", plog.Args("gitlabs", cacheOpts.Gitlabs))
+	plog.Debug("Remotes Loaded", plog.Args("remotes", cacheOpts.Remotes))
+}
+
+func getGitLabRemotes(cmd *cobra.Command) []remote.Remote {
+	gitRemotes := make([]remote.Remote, 0)
+
+	// Support legacy keys
+	if conf.GitlabHost != "" && conf.GitlabToken != "" {
+		conf.Gitlabs = append(conf.Gitlabs, config.GitlabConfig{
+			Host:       conf.GitlabHost,
+			Name:       conf.GitlabHost,
+			Token:      conf.GitlabToken,
+			CloneProto: config.CloneProtoSSH,
+		})
+	}
+
+	// Load Gitlabs
+	for _, gl := range conf.Gitlabs {
+		gitlabRemote, err := gitlabremote.NewGitlabRemote(&remote.RemoteInfo{
+			Ctx:        cmd.Context(),
+			Host:       gl.Host,
+			Name:       gl.Name,
+			Token:      gl.Token,
+			CloneProto: gl.CloneProto,
+		})
+		if err != nil {
+			plog.Error("Failed to prepare GitLab remote", plog.Args("error", err))
+		} else {
+			gitRemotes = append(gitRemotes, gitlabRemote)
+		}
+	}
+
+	return gitRemotes
 }
 
 func postProjectCache(cmd *cobra.Command, args []string) {

internal/cache/cache.go

@@ -10,7 +10,6 @@ import (
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
-	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
 	"gopkg.in/yaml.v3"
 )
 
@@ -26,8 +25,7 @@ type Cache struct {
 	file         string
 	log          *pterm.Logger
 	path         string
-	gitlabs      *remotes.Clients
-	remotes      *remote.Remotes
+	remotes      *remotes.Remotes
 }
 
 type CacheOpts struct {
@@ -35,8 +33,7 @@ type CacheOpts struct {
 	ProjectsPath string
 	TTL          time.Duration
 	Logger       *pterm.Logger
-	Gitlabs      *remotes.Clients
-	Remotes      *remote.Remotes
+	Remotes      *remotes.Remotes
 	Config       *config.Config
 }
 
@@ -148,7 +145,7 @@ func (c *Cache) refresh() {
 	// For backwards-compatibility only
 	c.setAliasRemotes()
 	// Retrieve and add/update projects
-	c.LoadGitlabs()
+	c.LoadRemotes()
 }
 
 // Iterates through all GitLab projects the user has access to, updating
@@ -164,10 +161,10 @@ func (c *Cache) String() string {
 		c.Updated.String(),
 		len(c.Projects),
 		len(c.Aliases),
-		len(*c.gitlabs),
+		len(*c.remotes),
 	)
-	for _, r := range *c.gitlabs {
-		cacheString += " " + r.Config.Host
+	for _, r := range *c.remotes {
+		cacheString += " " + r.GetInfo().Host
 	}
 	return cacheString
 }
@@ -196,14 +193,6 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) {
 		err = createProjectCache(opts.Path)
 	}
 
-	// Combine old-and-new gitlabs
-	var gitlabs *remotes.Clients
-	if opts.Gitlabs != nil {
-		gitlabs = opts.Gitlabs
-	} else {
-		gitlabs = remotes.NewCLients()
-	}
-
 	cache := &Cache{
 		Projects:    make([]*projects.Project, 0),
 		Aliases:     make([]*ProjectAlias, 0),
@@ -213,8 +202,7 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) {
 		lock:        &sync.Mutex{},
 		contentLock: &sync.Mutex{},
 		log:         opts.Logger,
-		gitlabs:     gitlabs,
-		remotes:     remote.NewRemotes(),
+		remotes:     opts.Remotes,
 		path:        opts.ProjectsPath,
 	}
 

internal/cache/cache_load.go

@@ -6,32 +6,43 @@ import (
 
 	"github.com/pterm/pterm"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote"
 )
 
-func (c *Cache) LoadGitlabs() {
+func (c *Cache) LoadRemotes() {
 	wg := &sync.WaitGroup{}
 	writer := pterm.DefaultMultiPrinter
 
-	for _, gl := range *c.gitlabs {
+	for _, r := range *c.remotes {
+		if !remote.IsAlive(r) {
+			c.log.Error("Skipping load of remote, not alive", c.log.Args(
+				"remote", r.GetInfo(),
+			))
+			continue
+		}
+
 		c.log.Info("Loading projects for remote", c.log.Args(
-			"host", gl.Config.Host,
-			"name", gl.Config.Name,
+			"host", r.GetInfo().Host,
+			"name", r.GetInfo().Name,
 		))
 
-		opts := *remotes.DefaultListOpts
-		opts.Owned = &c.config.Cache.Load.OwnerOnly
-		projects := gl.GetTotalProjects(&opts)
+		opts := &remote.RemoteQueryOpts{
+			Ctx:       r.GetInfo().Ctx,
+			OwnerOnly: c.config.Cache.Load.OwnerOnly,
+		}
+		pi := remotes.StreamRemote(r, opts)
 
 		// Prepare progressbar
 		pBar, _ := pterm.DefaultProgressbar.
 			WithShowPercentage(true).
-			WithTotal(projects).
+			WithTotal(pi.NumProjects).
 			WithWriter(writer.NewWriter()).
 			WithMaxWidth(100).
-			Start(gl.Config.Host)
+			Start(r.GetInfo().Name)
 
 		wg.Add(1)
-		go c.LoadGitlab(gl, wg, pBar, projects)
+		go c.ReceiveRemoteStream(r, wg, pBar, pi)
 	}
 
 	fmt.Println("")
@@ -43,10 +54,8 @@ func (c *Cache) LoadGitlabs() {
 	fmt.Println("")
 }
 
-func (c *Cache) LoadRemote(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) {
+func (c *Cache) ReceiveRemoteStream(remote remote.Remote, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, progressInfo *load.ProgressInfo) {
 	defer wg.Done()
-	progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects)
-
 	for {
 		select {
 		case p := <-progressInfo.ProgressChan:
@@ -54,30 +63,9 @@ func (c *Cache) LoadRemote(client *remotes.Client, wg *sync.WaitGroup, pBar *pte
 		case p := <-progressInfo.ProjectsChan:
 			c.AddProjects(p...)
 		case e := <-progressInfo.ErrorChan:
-			c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name))
-		case <-client.Ctx.Done():
-			c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err()))
-			return
-		case <-progressInfo.DoneChan:
-			return
-		}
-	}
-}
-
-func (c *Cache) LoadGitlab(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) {
-	defer wg.Done()
-	progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects)
-
-	for {
-		select {
-		case p := <-progressInfo.ProgressChan:
-			pBar.Add(p.Projects)
-		case p := <-progressInfo.ProjectsChan:
-			c.AddProjects(p...)
-		case e := <-progressInfo.ErrorChan:
-			c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name))
-		case <-client.Ctx.Done():
-			c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err()))
+			c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", remote.GetInfo().Name))
+		case <-remote.GetInfo().Ctx.Done():
+			c.log.Warn("LoadProjects cancelled", c.log.Args("reason", remote.GetInfo().Ctx.Err()))
 			return
 		case <-progressInfo.DoneChan:
 			return

internal/cache/projects_git.go

@@ -31,7 +31,7 @@ func (c *Cache) OpenProject(ctx context.Context, project *projects.Project) *git
 		// Check to make sure we can connect before we time out
 		// shouldn't be necessary, but go-git does not properly
 		// honor its context
-		if err := project.CheckHost(projects.GitlabProtoSSH); err != nil {
+		if err := project.CheckHost(projects.GitProtoSSH); err != nil {
 			c.log.Fatal("Git remote unreachable, giving up", c.log.Args("error", err))
 		}
 

internal/config/config.go

@@ -22,14 +22,23 @@ type GiteaConfig struct {
 	Host       string     `yaml:"host" json:"host"`
 	Name       string     `yaml:"name" json:"name"`
 	Token      string     `yaml:"token" json:"token"`
+	CloneProto CloneProto `yaml:"cloneProto" json:"cloneProto"`
 }
 
 type GitlabConfig struct {
 	Host       string     `yaml:"host" json:"host"`
 	Name       string     `yaml:"name" json:"name"`
 	Token      string     `yaml:"token" json:"token"`
+	CloneProto CloneProto `yaml:"cloneProto" json:"cloneProto"`
 }
 
+type CloneProto string
+
+const (
+	CloneProtoSSH  CloneProto = "ssh"
+	CloneProtoHTTP CloneProto = "http"
+)
+
 type editorConfig struct {
 	DisplayName   string `yaml:"displanName,omitempty" json:"displanName,omitempty"`
 	Binary        string `yaml:"binary,omitempty" json:"binary,omitempty"`
@@ -59,6 +68,7 @@ var DefaultConfig = Config{
 	Gitlabs: []GitlabConfig{{
 		Host:       "https://gitlab.com",
 		Token:      "yourtokenhere",
+		CloneProto: CloneProtoSSH,
 		Name:       "GitLab",
 	}},
 	Cache: cacheConfig{

internal/remotes/gitlab/gitlab_api.go

@@ -18,6 +18,7 @@ func NewGitlabApi(info *remote.RemoteInfo) (*gitlab.Client, error) {
 func (r *GitlabRemote) GetNumProjects(opts *remote.RemoteQueryOpts) int {
 	listOpts := *DefaultListOpts
 	listOpts.PerPage = 1
+	listOpts.Page = 1
 	listOpts.Simple = gitlab.Ptr[bool](true)
 	_, resp, err := r.api.Projects.ListProjects(&listOpts)
 	if err != nil {

internal/remotes/load/load.go

@@ -1,8 +1,6 @@
 package load
 
-import (
-	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
-)
+import "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
 
 // This package provides structs that serve
 // as the interface between remotes, and any code

internal/remotes/projects/projects_net.go

@@ -13,22 +13,22 @@ import (
 
 const defNetDialTimeoutSecs = 5
 
-type GitlabProto int
+type GitProto int
 
 const (
-	GitlabProtoSSH GitlabProto = iota
-	GitlabProtoHTTP
+	GitProtoSSH GitProto = iota
+	GitProtoHTTP
 )
 
 var (
 	ErrUnknownHost error = errors.New("No addresses found for host")
 )
 
-func (p *Project) CheckHost(proto GitlabProto) error {
+func (p *Project) CheckHost(proto GitProto) error {
 	switch proto {
-	case GitlabProtoHTTP:
+	case GitProtoHTTP:
 		return p.checkHTTPRemote()
-	case GitlabProtoSSH:
+	case GitProtoSSH:
 		return p.checkSSHRemote()
 	}
 	return errors.New("Unknown git protocol")

internal/remotes/remote/remote.go

@@ -2,7 +2,11 @@ package remote
 
 import (
 	"context"
+	"fmt"
+	"net"
+	"time"
 
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load"
 )
 
@@ -11,15 +15,10 @@ type RemoteInfo struct {
 	Host       string
 	Name       string
 	Token      string
+	CloneProto config.CloneProto
 }
 
-type RemoteProto string
-
-const (
-	RemoteProtoSSH   RemoteProto = "ssh"
-	RemoteProtoHTTP  RemoteProto = "http"
-	RemoteProtoHTTPS RemoteProto = "https"
-)
+const defNetDialTimeoutSecs = 3
 
 // Any remote needs to be able to return
 // the number of projects the user has access to and also
@@ -27,7 +26,31 @@ const (
 // provided by *load.ProgressInfo
 type Remote interface {
 	GetInfo() *RemoteInfo                                // Returns basic RemoteInfo struct
-	IsAlive(RemoteProto) bool                            // Indicates if the remote is reachable by either SSH or HTTP
 	GetNumProjects(*RemoteQueryOpts) int                 // Returns total number of accessible projects
 	StreamProjects(*load.ProgressInfo, *RemoteQueryOpts) // Streams projects to chans provided in load.ProgressInfo
 }
+
+func IsAlive(remote Remote) bool {
+	var port int
+	switch remote.GetInfo().CloneProto {
+	case config.CloneProtoHTTP:
+		port = 443
+	case config.CloneProtoSSH:
+		port = 22
+	}
+
+	d, err := net.DialTimeout("tcp",
+		fmt.Sprintf("%s:%d", remote.GetInfo().Host, port),
+		defNetDialTimeoutSecs*time.Second)
+
+	if err == nil {
+		_, err = d.Write([]byte("ok"))
+		d.Close()
+	}
+
+	if err == nil {
+		return true
+	} else {
+		return false
+	}
+}

internal/remotes/remote/remote_opts.go

@@ -2,6 +2,8 @@ package remote
 
 import "context"
 
+// Generic options to be passed to any
+// impelenter of the Remote interface
 type RemoteQueryOpts struct {
 	Ctx       context.Context
 	OwnerOnly bool

internal/remotes/remotes.go

@@ -17,8 +17,8 @@ const (
 
 type Remotes []remote.Remote
 
-func (r *Remotes) AddRemote(remote remote.Remote) {
-	*r = append(*r, remote)
+func (r *Remotes) AddRemotes(remote ...remote.Remote) {
+	*r = append(*r, remote...)
 }
 
 func (r *Remotes) GetRemoteByHost(host string) remote.Remote {
@@ -31,20 +31,18 @@ func (r *Remotes) GetRemoteByHost(host string) remote.Remote {
 }
 
 // Launches project streamsers for all remotes in goroutines
-// returns slice of load.ProgressInfo
-func (r *Remotes) StreamRemotes(opts *remote.RemoteQueryOpts) []*load.ProgressInfo {
-	progressInfos := make([]*load.ProgressInfo, len(*r))
-	for i, remoteInstance := range *r {
-		progressInfos[i] = &load.ProgressInfo{
+// returns slice of load.ProgressInfo, does not block, streamer is
+// launched in goroutine
+func StreamRemote(r remote.Remote, opts *remote.RemoteQueryOpts) *load.ProgressInfo {
+	progressInfo := &load.ProgressInfo{
 		ProgressChan: make(chan load.Progress),
 		ProjectsChan: make(chan []*projects.Project),
 		ErrorChan:    make(chan error),
 		DoneChan:     make(chan interface{}),
-			NumProjects:  remoteInstance.GetNumProjects(opts),
+		NumProjects:  r.GetNumProjects(opts),
 	}
-		go remoteInstance.StreamProjects(progressInfos[i], opts)
-	}
-	return progressInfos
+	go r.StreamProjects(progressInfo, opts)
+	return progressInfo
 }
 
 func NewRemotes() *Remotes {

internal/remotes/remotes_client.go

@@ -1,143 +0,0 @@
-package remotes
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"github.com/pterm/pterm"
-	"github.com/xanzy/go-gitlab"
-	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config"
-	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects"
-)
-
-type Client struct {
-	Ctx       context.Context
-	Config    *config.GitlabConfig
-	apiClient *gitlab.Client
-}
-
-type Clients []*Client
-
-type ClientOpts struct {
-	Ctx   context.Context
-	Name  string
-	Host  string
-	Token string
-}
-
-func (c *Clients) AddClients(gitlabClient ...*Client) error {
-	var err error
-	for _, client := range gitlabClient {
-		if c.GetClientByHost(client.Config.Host) != nil {
-			err = errors.Join(err, fmt.Errorf("Client with host %s already exists", client.Config.Host))
-		} else {
-			*c = append(*c, client)
-		}
-	}
-	return err
-}
-
-func (c *Clients) GetClientByHost(host string) *Client {
-	for _, client := range *c {
-		if client.Config.Host == host {
-			return client
-		}
-	}
-	return nil
-}
-
-func NewCLients() *Clients {
-	var clients Clients
-	clients = make([]*Client, 0)
-	return &clients
-}
-
-func NewGitlabClients(clientOpts []*ClientOpts) (*Clients, error) {
-	var err error
-	clients := NewCLients()
-	for _, opts := range clientOpts {
-		gitlabClient, e := NewGitlabClient(opts)
-		if e != nil {
-			err = errors.Join(err, e)
-			continue
-		}
-		err = errors.Join(err, clients.AddClients(gitlabClient))
-	}
-	return clients, err
-}
-
-func NewGitlabClient(opts *ClientOpts) (*Client, error) {
-	client, err := gitlab.NewClient(opts.Token, gitlab.WithBaseURL(opts.Host))
-	if err != nil {
-		return nil, err
-	}
-	gitlabClient := &Client{
-		Ctx: opts.Ctx,
-		Config: &config.GitlabConfig{
-			Name:  opts.Name,
-			Host:  opts.Host,
-			Token: opts.Token,
-		},
-		apiClient: client,
-	}
-	return gitlabClient, nil
-}
-
-func (c *Client) Api() *gitlab.Client {
-	return c.apiClient
-}
-
-func (c *Client) GetTotalProjects(opts *gitlab.ListProjectsOptions) int {
-	reqOpts := *opts
-	reqOpts.ListOptions = gitlab.ListOptions{
-		Page:    1,
-		PerPage: 1,
-	}
-
-	var projects int
-	if _, r, e := c.apiClient.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil {
-		projects = r.TotalItems
-	}
-
-	return projects
-}
-
-// Returns a list of projects along with the next page and an error
-// if there was an error
-func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) (
-	[]*projects.Project, *gitlab.Response, error) {
-	pList := make([]*projects.Project, 0)
-	projects, resp, err := c.apiClient.Projects.ListProjects(
-		opts,
-		gitlab.WithContext(c.Ctx),
-	)
-	if err == nil {
-		pList = append(pList, c.handleProjects(projects)...)
-	}
-	return pList, resp, err
-}
-
-// A nil return indicates an API error or GitLab doesn't know what
-// language the project uses.
-func (c *Client) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages {
-	l, _, e := c.apiClient.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(c.Ctx))
-	if e != nil {
-		pterm.Error.Printfln("Failed requesting project languages: %s", e.Error())
-		return nil
-	}
-
-	var pLangs projects.ProjectLanguages
-	pLangs = make([]*projects.ProjectLanguage, len(*l))
-
-	var i int
-	for name, pcnt := range *l {
-		pLangs[i] = &projects.ProjectLanguage{
-			Name:       name,
-			Percentage: pcnt,
-		}
-		i++
-	}
-
-	return &pLangs
-}