diff --git a/cmd/config_generate.go b/cmd/config_generate.go index 6e1e492..c757dfd 100644 --- a/cmd/config_generate.go +++ b/cmd/config_generate.go @@ -101,6 +101,17 @@ func promptConfigSettings(c *config.Config) *config.Config { gitlabConfig.Token = token } + if proto, err := pterm.DefaultInteractiveSelect. + WithOptions([]string{string(config.CloneProtoHTTP), string(config.CloneProtoSSH)}). + WithDefaultText("Git Clone Protocol"). + Show(); err == nil { + if proto == "ssh" { + gitlabConfig.CloneProto = config.CloneProtoSSH + } else { + gitlabConfig.CloneProto = config.CloneProtoHTTP + } + } + if pPath, err := pterm.DefaultInteractiveTextInput. WithDefaultValue(c.ProjectPath). WithDefaultText("Enter path for projects and cache"). diff --git a/cmd/util_init.go b/cmd/util_init.go index 9624b63..736aa31 100644 --- a/cmd/util_init.go +++ b/cmd/util_init.go @@ -10,6 +10,8 @@ import ( "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/cache" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes" + gitlabremote "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/gitlab" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" "golang.org/x/sys/unix" ) @@ -19,52 +21,19 @@ import ( // func from their PersistentPreRun commands func initProjectCache(cmd *cobra.Command, args []string) { + var err error plog.Debug("Running pre-run for cacheCmd") conf.Cache.File = conf.ProjectPath + "/.cache.yaml" - // Backwards-compatible support for singular instance - opts := make([]*remotes.ClientOpts, 0) - - if conf.GitlabHost != "" { - opts = append(opts, &remotes.ClientOpts{ - Ctx: cmd.Context(), - Host: conf.GitlabHost, // deprecated, switch to gitlabs - Token: conf.GitlabToken, // deprecated, switch to gitlabs - Name: conf.GitlabHost, // not originally supported, use the new gitlabs field - }) - } - - // If defined, load additional instances - for _, g := range conf.Gitlabs { - opts = append(opts, &remotes.ClientOpts{ - Ctx: cmd.Context(), - Name: g.Name, - Host: g.Host, - Token: g.Token, - }) - } - - // We need at least one GitLab - if len(opts) < 1 { - plog.Error("At least one GitLab must be configured. Add to .gitlabs in your config file") - os.Exit(1) - } - - // Load all gitlab configs into clients - var gitlabs *remotes.Clients - var err error - gitlabs, err = remotes.NewGitlabClients(opts) - if err != nil { - plog.Error("Failed to create GitLab clients", plog.Args("error", err)) - os.Exit(1) - } + gitRemotes := remotes.NewRemotes() + gitRemotes.AddRemotes(getGitLabRemotes(cmd)...) cacheOpts := &cache.CacheOpts{ ProjectsPath: conf.ProjectPath, Path: conf.Cache.File, TTL: conf.Cache.Ttl, Logger: plog, - Gitlabs: gitlabs, + Remotes: gitRemotes, Config: &conf, } if projectCache, err = cache.NewProjectCache(cacheOpts); err != nil { @@ -77,7 +46,39 @@ func initProjectCache(cmd *cobra.Command, args []string) { os.Exit(1) } - plog.Debug("Gitlab Clients", plog.Args("gitlabs", cacheOpts.Gitlabs)) + plog.Debug("Remotes Loaded", plog.Args("remotes", cacheOpts.Remotes)) +} + +func getGitLabRemotes(cmd *cobra.Command) []remote.Remote { + gitRemotes := make([]remote.Remote, 0) + + // Support legacy keys + if conf.GitlabHost != "" && conf.GitlabToken != "" { + conf.Gitlabs = append(conf.Gitlabs, config.GitlabConfig{ + Host: conf.GitlabHost, + Name: conf.GitlabHost, + Token: conf.GitlabToken, + CloneProto: config.CloneProtoSSH, + }) + } + + // Load Gitlabs + for _, gl := range conf.Gitlabs { + gitlabRemote, err := gitlabremote.NewGitlabRemote(&remote.RemoteInfo{ + Ctx: cmd.Context(), + Host: gl.Host, + Name: gl.Name, + Token: gl.Token, + CloneProto: gl.CloneProto, + }) + if err != nil { + plog.Error("Failed to prepare GitLab remote", plog.Args("error", err)) + } else { + gitRemotes = append(gitRemotes, gitlabRemote) + } + } + + return gitRemotes } func postProjectCache(cmd *cobra.Command, args []string) { diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 6f6e072..51421b4 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -10,7 +10,6 @@ import ( "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" - "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" "gopkg.in/yaml.v3" ) @@ -26,8 +25,7 @@ type Cache struct { file string log *pterm.Logger path string - gitlabs *remotes.Clients - remotes *remote.Remotes + remotes *remotes.Remotes } type CacheOpts struct { @@ -35,8 +33,7 @@ type CacheOpts struct { ProjectsPath string TTL time.Duration Logger *pterm.Logger - Gitlabs *remotes.Clients - Remotes *remote.Remotes + Remotes *remotes.Remotes Config *config.Config } @@ -148,7 +145,7 @@ func (c *Cache) refresh() { // For backwards-compatibility only c.setAliasRemotes() // Retrieve and add/update projects - c.LoadGitlabs() + c.LoadRemotes() } // Iterates through all GitLab projects the user has access to, updating @@ -164,10 +161,10 @@ func (c *Cache) String() string { c.Updated.String(), len(c.Projects), len(c.Aliases), - len(*c.gitlabs), + len(*c.remotes), ) - for _, r := range *c.gitlabs { - cacheString += " " + r.Config.Host + for _, r := range *c.remotes { + cacheString += " " + r.GetInfo().Host } return cacheString } @@ -196,14 +193,6 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) { err = createProjectCache(opts.Path) } - // Combine old-and-new gitlabs - var gitlabs *remotes.Clients - if opts.Gitlabs != nil { - gitlabs = opts.Gitlabs - } else { - gitlabs = remotes.NewCLients() - } - cache := &Cache{ Projects: make([]*projects.Project, 0), Aliases: make([]*ProjectAlias, 0), @@ -213,8 +202,7 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) { lock: &sync.Mutex{}, contentLock: &sync.Mutex{}, log: opts.Logger, - gitlabs: gitlabs, - remotes: remote.NewRemotes(), + remotes: opts.Remotes, path: opts.ProjectsPath, } diff --git a/internal/cache/cache_load.go b/internal/cache/cache_load.go index c863609..23cc0af 100644 --- a/internal/cache/cache_load.go +++ b/internal/cache/cache_load.go @@ -6,32 +6,43 @@ import ( "github.com/pterm/pterm" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/remote" ) -func (c *Cache) LoadGitlabs() { +func (c *Cache) LoadRemotes() { wg := &sync.WaitGroup{} writer := pterm.DefaultMultiPrinter - for _, gl := range *c.gitlabs { + for _, r := range *c.remotes { + if !remote.IsAlive(r) { + c.log.Error("Skipping load of remote, not alive", c.log.Args( + "remote", r.GetInfo(), + )) + continue + } + c.log.Info("Loading projects for remote", c.log.Args( - "host", gl.Config.Host, - "name", gl.Config.Name, + "host", r.GetInfo().Host, + "name", r.GetInfo().Name, )) - opts := *remotes.DefaultListOpts - opts.Owned = &c.config.Cache.Load.OwnerOnly - projects := gl.GetTotalProjects(&opts) + opts := &remote.RemoteQueryOpts{ + Ctx: r.GetInfo().Ctx, + OwnerOnly: c.config.Cache.Load.OwnerOnly, + } + pi := remotes.StreamRemote(r, opts) // Prepare progressbar pBar, _ := pterm.DefaultProgressbar. WithShowPercentage(true). - WithTotal(projects). + WithTotal(pi.NumProjects). WithWriter(writer.NewWriter()). WithMaxWidth(100). - Start(gl.Config.Host) + Start(r.GetInfo().Name) wg.Add(1) - go c.LoadGitlab(gl, wg, pBar, projects) + go c.ReceiveRemoteStream(r, wg, pBar, pi) } fmt.Println("") @@ -43,10 +54,8 @@ func (c *Cache) LoadGitlabs() { fmt.Println("") } -func (c *Cache) LoadRemote(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) { +func (c *Cache) ReceiveRemoteStream(remote remote.Remote, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, progressInfo *load.ProgressInfo) { defer wg.Done() - progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects) - for { select { case p := <-progressInfo.ProgressChan: @@ -54,30 +63,9 @@ func (c *Cache) LoadRemote(client *remotes.Client, wg *sync.WaitGroup, pBar *pte case p := <-progressInfo.ProjectsChan: c.AddProjects(p...) case e := <-progressInfo.ErrorChan: - c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name)) - case <-client.Ctx.Done(): - c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err())) - return - case <-progressInfo.DoneChan: - return - } - } -} - -func (c *Cache) LoadGitlab(client *remotes.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) { - defer wg.Done() - progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects) - - for { - select { - case p := <-progressInfo.ProgressChan: - pBar.Add(p.Projects) - case p := <-progressInfo.ProjectsChan: - c.AddProjects(p...) - case e := <-progressInfo.ErrorChan: - c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", client.Config.Name)) - case <-client.Ctx.Done(): - c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err())) + c.log.Error("Fetch projects error", c.log.Args("error", e, "remote", remote.GetInfo().Name)) + case <-remote.GetInfo().Ctx.Done(): + c.log.Warn("LoadProjects cancelled", c.log.Args("reason", remote.GetInfo().Ctx.Err())) return case <-progressInfo.DoneChan: return diff --git a/internal/cache/projects_git.go b/internal/cache/projects_git.go index e437767..6b562af 100644 --- a/internal/cache/projects_git.go +++ b/internal/cache/projects_git.go @@ -31,7 +31,7 @@ func (c *Cache) OpenProject(ctx context.Context, project *projects.Project) *git // Check to make sure we can connect before we time out // shouldn't be necessary, but go-git does not properly // honor its context - if err := project.CheckHost(projects.GitlabProtoSSH); err != nil { + if err := project.CheckHost(projects.GitProtoSSH); err != nil { c.log.Fatal("Git remote unreachable, giving up", c.log.Args("error", err)) } diff --git a/internal/config/config.go b/internal/config/config.go index 9b15903..638d70f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,17 +19,26 @@ type Config struct { } type GiteaConfig struct { - Host string `yaml:"host" json:"host"` - Name string `yaml:"name" json:"name"` - Token string `yaml:"token" json:"token"` + Host string `yaml:"host" json:"host"` + Name string `yaml:"name" json:"name"` + Token string `yaml:"token" json:"token"` + CloneProto CloneProto `yaml:"cloneProto" json:"cloneProto"` } type GitlabConfig struct { - Host string `yaml:"host" json:"host"` - Name string `yaml:"name" json:"name"` - Token string `yaml:"token" json:"token"` + Host string `yaml:"host" json:"host"` + Name string `yaml:"name" json:"name"` + Token string `yaml:"token" json:"token"` + CloneProto CloneProto `yaml:"cloneProto" json:"cloneProto"` } +type CloneProto string + +const ( + CloneProtoSSH CloneProto = "ssh" + CloneProtoHTTP CloneProto = "http" +) + type editorConfig struct { DisplayName string `yaml:"displanName,omitempty" json:"displanName,omitempty"` Binary string `yaml:"binary,omitempty" json:"binary,omitempty"` @@ -57,9 +66,10 @@ var DefaultConfig = Config{ LogLevel: "warn", ProjectPath: "~/work/projects", Gitlabs: []GitlabConfig{{ - Host: "https://gitlab.com", - Token: "yourtokenhere", - Name: "GitLab", + Host: "https://gitlab.com", + Token: "yourtokenhere", + CloneProto: CloneProtoSSH, + Name: "GitLab", }}, Cache: cacheConfig{ Ttl: 168 * time.Hour, diff --git a/internal/remotes/gitlab/gitlab_api.go b/internal/remotes/gitlab/gitlab_api.go index ce140a4..c929f55 100644 --- a/internal/remotes/gitlab/gitlab_api.go +++ b/internal/remotes/gitlab/gitlab_api.go @@ -18,6 +18,7 @@ func NewGitlabApi(info *remote.RemoteInfo) (*gitlab.Client, error) { func (r *GitlabRemote) GetNumProjects(opts *remote.RemoteQueryOpts) int { listOpts := *DefaultListOpts listOpts.PerPage = 1 + listOpts.Page = 1 listOpts.Simple = gitlab.Ptr[bool](true) _, resp, err := r.api.Projects.ListProjects(&listOpts) if err != nil { diff --git a/internal/remotes/load/load.go b/internal/remotes/load/load.go index 3c5f634..324314c 100644 --- a/internal/remotes/load/load.go +++ b/internal/remotes/load/load.go @@ -1,8 +1,6 @@ package load -import ( - "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" -) +import "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" // This package provides structs that serve // as the interface between remotes, and any code diff --git a/internal/remotes/projects/projects_net.go b/internal/remotes/projects/projects_net.go index 686bb4d..ddc0d6a 100644 --- a/internal/remotes/projects/projects_net.go +++ b/internal/remotes/projects/projects_net.go @@ -13,22 +13,22 @@ import ( const defNetDialTimeoutSecs = 5 -type GitlabProto int +type GitProto int const ( - GitlabProtoSSH GitlabProto = iota - GitlabProtoHTTP + GitProtoSSH GitProto = iota + GitProtoHTTP ) var ( ErrUnknownHost error = errors.New("No addresses found for host") ) -func (p *Project) CheckHost(proto GitlabProto) error { +func (p *Project) CheckHost(proto GitProto) error { switch proto { - case GitlabProtoHTTP: + case GitProtoHTTP: return p.checkHTTPRemote() - case GitlabProtoSSH: + case GitProtoSSH: return p.checkSSHRemote() } return errors.New("Unknown git protocol") diff --git a/internal/remotes/remote/remote.go b/internal/remotes/remote/remote.go index c86fdd2..c30e598 100644 --- a/internal/remotes/remote/remote.go +++ b/internal/remotes/remote/remote.go @@ -2,24 +2,23 @@ package remote import ( "context" + "fmt" + "net" + "time" + "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config" "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/load" ) type RemoteInfo struct { - Ctx context.Context - Host string - Name string - Token string + Ctx context.Context + Host string + Name string + Token string + CloneProto config.CloneProto } -type RemoteProto string - -const ( - RemoteProtoSSH RemoteProto = "ssh" - RemoteProtoHTTP RemoteProto = "http" - RemoteProtoHTTPS RemoteProto = "https" -) +const defNetDialTimeoutSecs = 3 // Any remote needs to be able to return // the number of projects the user has access to and also @@ -27,7 +26,31 @@ const ( // provided by *load.ProgressInfo type Remote interface { GetInfo() *RemoteInfo // Returns basic RemoteInfo struct - IsAlive(RemoteProto) bool // Indicates if the remote is reachable by either SSH or HTTP GetNumProjects(*RemoteQueryOpts) int // Returns total number of accessible projects StreamProjects(*load.ProgressInfo, *RemoteQueryOpts) // Streams projects to chans provided in load.ProgressInfo } + +func IsAlive(remote Remote) bool { + var port int + switch remote.GetInfo().CloneProto { + case config.CloneProtoHTTP: + port = 443 + case config.CloneProtoSSH: + port = 22 + } + + d, err := net.DialTimeout("tcp", + fmt.Sprintf("%s:%d", remote.GetInfo().Host, port), + defNetDialTimeoutSecs*time.Second) + + if err == nil { + _, err = d.Write([]byte("ok")) + d.Close() + } + + if err == nil { + return true + } else { + return false + } +} diff --git a/internal/remotes/remote/remote_opts.go b/internal/remotes/remote/remote_opts.go index 3b10ecf..578608b 100644 --- a/internal/remotes/remote/remote_opts.go +++ b/internal/remotes/remote/remote_opts.go @@ -2,6 +2,8 @@ package remote import "context" +// Generic options to be passed to any +// impelenter of the Remote interface type RemoteQueryOpts struct { Ctx context.Context OwnerOnly bool diff --git a/internal/remotes/remotes.go b/internal/remotes/remotes.go index 2a5b6f9..46091b0 100644 --- a/internal/remotes/remotes.go +++ b/internal/remotes/remotes.go @@ -17,8 +17,8 @@ const ( type Remotes []remote.Remote -func (r *Remotes) AddRemote(remote remote.Remote) { - *r = append(*r, remote) +func (r *Remotes) AddRemotes(remote ...remote.Remote) { + *r = append(*r, remote...) } func (r *Remotes) GetRemoteByHost(host string) remote.Remote { @@ -31,20 +31,18 @@ func (r *Remotes) GetRemoteByHost(host string) remote.Remote { } // Launches project streamsers for all remotes in goroutines -// returns slice of load.ProgressInfo -func (r *Remotes) StreamRemotes(opts *remote.RemoteQueryOpts) []*load.ProgressInfo { - progressInfos := make([]*load.ProgressInfo, len(*r)) - for i, remoteInstance := range *r { - progressInfos[i] = &load.ProgressInfo{ - ProgressChan: make(chan load.Progress), - ProjectsChan: make(chan []*projects.Project), - ErrorChan: make(chan error), - DoneChan: make(chan interface{}), - NumProjects: remoteInstance.GetNumProjects(opts), - } - go remoteInstance.StreamProjects(progressInfos[i], opts) +// returns slice of load.ProgressInfo, does not block, streamer is +// launched in goroutine +func StreamRemote(r remote.Remote, opts *remote.RemoteQueryOpts) *load.ProgressInfo { + progressInfo := &load.ProgressInfo{ + ProgressChan: make(chan load.Progress), + ProjectsChan: make(chan []*projects.Project), + ErrorChan: make(chan error), + DoneChan: make(chan interface{}), + NumProjects: r.GetNumProjects(opts), } - return progressInfos + go r.StreamProjects(progressInfo, opts) + return progressInfo } func NewRemotes() *Remotes { diff --git a/internal/remotes/remotes_client.go b/internal/remotes/remotes_client.go deleted file mode 100644 index e2f8f2a..0000000 --- a/internal/remotes/remotes_client.go +++ /dev/null @@ -1,143 +0,0 @@ -package remotes - -import ( - "context" - "errors" - "fmt" - - "github.com/pterm/pterm" - "github.com/xanzy/go-gitlab" - "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/config" - "gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/remotes/projects" -) - -type Client struct { - Ctx context.Context - Config *config.GitlabConfig - apiClient *gitlab.Client -} - -type Clients []*Client - -type ClientOpts struct { - Ctx context.Context - Name string - Host string - Token string -} - -func (c *Clients) AddClients(gitlabClient ...*Client) error { - var err error - for _, client := range gitlabClient { - if c.GetClientByHost(client.Config.Host) != nil { - err = errors.Join(err, fmt.Errorf("Client with host %s already exists", client.Config.Host)) - } else { - *c = append(*c, client) - } - } - return err -} - -func (c *Clients) GetClientByHost(host string) *Client { - for _, client := range *c { - if client.Config.Host == host { - return client - } - } - return nil -} - -func NewCLients() *Clients { - var clients Clients - clients = make([]*Client, 0) - return &clients -} - -func NewGitlabClients(clientOpts []*ClientOpts) (*Clients, error) { - var err error - clients := NewCLients() - for _, opts := range clientOpts { - gitlabClient, e := NewGitlabClient(opts) - if e != nil { - err = errors.Join(err, e) - continue - } - err = errors.Join(err, clients.AddClients(gitlabClient)) - } - return clients, err -} - -func NewGitlabClient(opts *ClientOpts) (*Client, error) { - client, err := gitlab.NewClient(opts.Token, gitlab.WithBaseURL(opts.Host)) - if err != nil { - return nil, err - } - gitlabClient := &Client{ - Ctx: opts.Ctx, - Config: &config.GitlabConfig{ - Name: opts.Name, - Host: opts.Host, - Token: opts.Token, - }, - apiClient: client, - } - return gitlabClient, nil -} - -func (c *Client) Api() *gitlab.Client { - return c.apiClient -} - -func (c *Client) GetTotalProjects(opts *gitlab.ListProjectsOptions) int { - reqOpts := *opts - reqOpts.ListOptions = gitlab.ListOptions{ - Page: 1, - PerPage: 1, - } - - var projects int - if _, r, e := c.apiClient.Projects.ListProjects(opts, gitlab.WithContext(c.Ctx)); e == nil { - projects = r.TotalItems - } - - return projects -} - -// Returns a list of projects along with the next page and an error -// if there was an error -func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) ( - []*projects.Project, *gitlab.Response, error) { - pList := make([]*projects.Project, 0) - projects, resp, err := c.apiClient.Projects.ListProjects( - opts, - gitlab.WithContext(c.Ctx), - ) - if err == nil { - pList = append(pList, c.handleProjects(projects)...) - } - return pList, resp, err -} - -// A nil return indicates an API error or GitLab doesn't know what -// language the project uses. -func (c *Client) GetProjectLanguages(project *gitlab.Project) *projects.ProjectLanguages { - l, _, e := c.apiClient.Projects.GetProjectLanguages(project.ID, gitlab.WithContext(c.Ctx)) - if e != nil { - pterm.Error.Printfln("Failed requesting project languages: %s", e.Error()) - return nil - } - - var pLangs projects.ProjectLanguages - pLangs = make([]*projects.ProjectLanguage, len(*l)) - - var i int - for name, pcnt := range *l { - pLangs[i] = &projects.ProjectLanguage{ - Name: name, - Percentage: pcnt, - } - i++ - } - - return &pLangs -}