rmcguire/git-project-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add multi-remote support for GitLab (#1)

Browse Source 
Co-authored-by: Ryan D McGuire <ryand_mcguire@sweetwater.com>
Reviewed-on: 50W/git-project-manager#1
This commit is contained in:
Ryan McGuire
2024-01-14 15:33:15 +00:00
parent 415290de20
commit b944af140a
27 changed files with 868 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
internal/projects/cache.go
View File

@ -3,7 +3,6 @@ package projects
import (
"fmt"
"os"
"strings"
"sync"
"time"
@ -19,12 +18,13 @@ type Cache struct {
Updated time.Time
config *config.Config
readFromFile bool
lock *sync.Mutex
lock *sync.Mutex // Lock the entire cache
contentLock *sync.Mutex // Lock projects / aliases
ttl time.Duration
file string
log *pterm.Logger
gitlab *gitlab.Client
path string
gitlabs *gitlab.Clients
}
type CacheOpts struct {
@ -32,7 +32,7 @@ type CacheOpts struct {
ProjectsPath string
TTL time.Duration
Logger *pterm.Logger
Gitlab *gitlab.Client
Gitlabs *gitlab.Clients
Config *config.Config
}
@ -94,6 +94,10 @@ func (c *Cache) write() {
func (c *Cache) Write() {
c.lock.Lock()
defer c.lock.Unlock()
c.log.Debug("Saving cache to disk", c.log.Args(
"projects", len(c.Projects),
"aliases", len(c.Aliases),
))
c.write()
}
@ -136,7 +140,11 @@ func (c *Cache) Clear(clearAliases bool) {
func (c *Cache) refresh() {
c.log.Info("Loading project cache, this may take a while\n")
defer c.setUpdated()
c.LoadProjects()
// Fix any dangling aliases
// For backwards-compatibility only
c.setAliasRemotes()
// Retrieve and add/update projects
c.LoadGitlabs()
}
// Iterates through all GitLab projects the user has access to, updating
@ -148,39 +156,16 @@ func (c *Cache) Refresh() {
}
func (c *Cache) String() string {
return fmt.Sprintf("Cache Updated %s: Projects %d, Aliases %d",
cacheString := fmt.Sprintf("Cache Updated %s: Projects %d, Aliases %d\nRemotes %d:",
c.Updated.String(),
len(c.Projects),
len(c.Aliases))
}
// This command will only dump projects that have
// been cloned locally. Setting all to true will list all projects
func (c *Cache) DumpString(all bool) string {
str := strings.Builder{}
var term string
if all {
term = pterm.Bold.Sprint("Projects")
} else {
term = pterm.Bold.Sprint("Local Projects")
len(c.Aliases),
len(*c.gitlabs),
)
for _, r := range *c.gitlabs {
cacheString += " " + r.Config.Host
}
str.WriteString(c.String() + "\n\n" + term + ":\n")
for _, project := range c.Projects {
if !all && !c.IsProjectCloned(project) {
continue
}
str.WriteString(" - " + pterm.FgLightBlue.Sprint(project.Name) + " (")
str.WriteString(project.PathWithNamespace + ")\n")
aliases := c.GetProjectAliases(project)
if len(aliases) > 0 {
str.WriteString(pterm.FgLightGreen.Sprint(" aliases:"))
for _, a := range aliases {
str.WriteString(" [" + pterm.FgCyan.Sprint(a.Alias) + "]")
}
str.WriteRune('\n')
}
}
return str.String()
return cacheString
}
func (c *Cache) setUpdated() {
@ -207,16 +192,25 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) {
err = createProjectCache(opts.Path)
}
// Combine old-and-new gitlabs
var gitlabs *gitlab.Clients
if opts.Gitlabs != nil {
gitlabs = opts.Gitlabs
} else {
gitlabs = gitlab.NewCLients()
}
cache := &Cache{
Projects: make([]*gitlab.Project, 0),
Aliases: make([]*ProjectAlias, 0),
config: opts.Config,
file: opts.Path,
ttl: opts.TTL,
lock: &sync.Mutex{},
log: opts.Logger,
gitlab: opts.Gitlab,
path: opts.ProjectsPath,
Projects: make([]*gitlab.Project, 0),
Aliases: make([]*ProjectAlias, 0),
config: opts.Config,
file: opts.Path,
ttl: opts.TTL,
lock: &sync.Mutex{},
contentLock: &sync.Mutex{},
log: opts.Logger,
gitlabs: gitlabs,
path: opts.ProjectsPath,
}
return cache, err

41
internal/projects/cache_aliases.go
View File

@ -2,8 +2,6 @@ package projects
import (
"errors"
"fmt"
"strings"
"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
"golang.org/x/exp/slices"
@ -23,7 +21,7 @@ func (c *Cache) DeleteAlias(alias *ProjectAlias) {
c.deleteAlias(alias)
}
func (c *Cache) addAlias(alias string, projectID int) error {
func (c *Cache) addAlias(alias string, projectID int, remote string) error {
if c.GetAliasByName(alias) != nil {
return errors.New("Failed to add alias, already exists")
}
@ -32,15 +30,16 @@ func (c *Cache) addAlias(alias string, projectID int) error {
&ProjectAlias{
Alias: alias,
ProjectID: projectID,
Remote: remote,
})
return nil
}
func (c *Cache) AddAlias(alias string, projectID int) error {
func (c *Cache) AddAlias(alias string, projectID int, remote string) error {
c.lock.Lock()
defer c.lock.Unlock()
return c.addAlias(alias, projectID)
return c.addAlias(alias, projectID, remote)
}
func (c *Cache) GetProjectsWithAliases() []*gitlab.Project {
@ -55,20 +54,26 @@ func (c *Cache) GetProjectsWithAliases() []*gitlab.Project {
return projectList
}
func (c *Cache) GetProjectAliasStrings(project *gitlab.Project) []string {
aliases := c.GetProjectAliases(project)
strings := make([]string, len(aliases))
for i, a := range c.GetProjectAliases(project) {
strings[i] = a.Alias
// This method only exists if a cache was built
// before multi-remote support. Upon the first load
// with multi-remotes, this will be run first to update
// any missing alias remotes
func (c *Cache) setAliasRemotes() {
for _, alias := range c.Aliases {
if alias.Remote == "" {
c.setAliasRemote(alias)
}
}
return strings
}
func (c *Cache) GetProjectStringWithAliases(project *gitlab.Project) string {
aliases := c.GetProjectAliasStrings(project)
return fmt.Sprintf("%s (%s) -> %s",
project.Name,
strings.Join(aliases, ", "),
project.PathWithNamespace,
)
func (c *Cache) setAliasRemote(alias *ProjectAlias) {
project := c.GetProjectByID(alias.ProjectID)
if project != nil {
alias.Remote = project.Remote
c.log.Debug("Fixed missing alias remote", c.log.Args(
"alias", alias.Alias,
"projectID", alias.ProjectID,
"remote", alias.Remote,
))
}
}

65
internal/projects/cache_load.go Normal file
View File

@ -0,0 +1,65 @@
package projects
import (
"fmt"
"sync"
"github.com/pterm/pterm"
"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
)
func (c *Cache) LoadGitlabs() {
wg := &sync.WaitGroup{}
writer := pterm.DefaultMultiPrinter
for _, gl := range *c.gitlabs {
c.log.Info("Loading projects for remote", c.log.Args(
"host", gl.Config.Host,
"name", gl.Config.Name,
))
opts := *gitlab.DefaultListOpts
opts.Owned = &c.config.Cache.Load.OwnerOnly
projects := gl.GetTotalProjects(&opts)
// Prepare progressbar
pBar, _ := pterm.DefaultProgressbar.
WithShowPercentage(true).
WithTotal(projects).
WithWriter(writer.NewWriter()).
WithMaxWidth(100).
Start(gl.Config.Host)
wg.Add(1)
go c.LoadGitlab(gl, wg, pBar, projects)
}
fmt.Println("")
writer.Start()
wg.Wait()
writer.Stop()
fmt.Println("")
}
func (c *Cache) LoadGitlab(client *gitlab.Client, wg *sync.WaitGroup, pBar *pterm.ProgressbarPrinter, projects int) {
defer wg.Done()
progressInfo := client.StreamProjects(c.config.Cache.Load.OwnerOnly, projects)
for {
select {
case p := <-progressInfo.ProgressChan:
pBar.Add(p.Projects)
case p := <-progressInfo.ProjectsChan:
c.AddProjects(p...)
case e := <-progressInfo.ErrorChan:
c.log.Error("Fetch GitLab projects error", c.log.Args("error", e, "gitlab", client.Config.Name))
case <-client.Ctx.Done():
c.log.Warn("LoadProjects cancelled", c.log.Args("reason", client.Ctx.Err()))
return
case <-progressInfo.DoneChan:
return
}
}
}

76
internal/projects/cache_projects.go Normal file
View File

@ -0,0 +1,76 @@
package projects
import (
"strings"
"github.com/pterm/pterm"
"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
"golang.org/x/exp/slices"
)
func (c *Cache) AddProjects(projects ...*gitlab.Project) {
c.contentLock.Lock()
defer c.contentLock.Unlock()
for _, p := range projects {
var existing *gitlab.Project
sameID := c.GetProjectsByID(p.ID)
// If there is only one by ID, we don't
// have to worry about matching remotes.
// If there are more than one, either match
// remotes or update the remote if it was never
// set due to being build from old code
// New caches should never have empty remotes.
if len(sameID) == 1 {
existing = sameID[0]
} else if len(sameID) > 1 {
for _, pr := range sameID {
if pr.Remote == p.Remote || pr.Remote == "" {
existing = pr
break
}
}
}
// Add a new one, or update if changed
if existing == nil {
c.Projects = append(c.Projects, p)
} else if *existing != *p {
*existing = *p
}
}
}
// This command will only dump projects that have
// been cloned locally. Setting all to true will list all projects
func (c *Cache) DumpString(all bool, search string, gitlabs ...string) string {
str := strings.Builder{}
var term string
if all {
term = pterm.Bold.Sprint("Projects")
} else {
term = pterm.Bold.Sprint("Local Projects")
}
str.WriteString(c.String() + "\n\n" + term + ":\n")
projects := c.FuzzyFindProjects(search)
for _, project := range projects {
if !all && !c.IsProjectCloned(project) {
continue
} else if len(gitlabs) > 0 && !slices.Contains(gitlabs, project.Remote) {
continue
}
str.WriteString(" - " + pterm.FgLightBlue.Sprint(project.Name) + " (")
str.WriteString(project.PathWithNamespace + ")\n")
aliases := c.GetProjectAliases(project)
if len(aliases) > 0 {
str.WriteString(pterm.FgLightGreen.Sprint(" aliases:"))
for _, a := range aliases {
str.WriteString(" [" + pterm.FgCyan.Sprint(a.Alias) + "]")
}
str.WriteRune('\n')
}
str.WriteString(pterm.FgLightMagenta.Sprint(" remote: "))
str.WriteString(project.Remote + "\n")
}
return str.String()
}

20
internal/projects/fuzz.go
View File

@ -4,6 +4,7 @@ import (
"strings"
"github.com/lithammer/fuzzysearch/fuzzy"
"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
)
// Performs a fuzzy find on the input string, returning the closest
@ -35,3 +36,22 @@ func (c *Cache) FuzzyFindAlias(name string) []*ProjectAlias {
}
return aliases
}
// Returns all matching projects by fuzzy find term
// Matches NameWithNamespace and Aliases
func (c *Cache) FuzzyFindProjects(search string) []*gitlab.Project {
projects := make([]*gitlab.Project, 0, len(c.Projects))
for _, p := range c.Projects {
if fuzzy.MatchFold(search, p.NameWithNamespace) {
projects = append(projects, p)
continue
}
for _, a := range c.GetProjectAliases(p) {
if fuzzy.MatchFold(search, a.Alias) {
projects = append(projects, p)
continue
}
}
}
return projects
}

129
internal/projects/projects.go
View File

@ -1,46 +1,13 @@
package projects
import (
"bytes"
"fmt"
"strings"
"text/tabwriter"
"github.com/pterm/pterm"
"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
"golang.org/x/exp/slices"
)
type ProjectAlias struct {
Alias string
ProjectID int
}
func ProjectAliasesString(aliases []*ProjectAlias) string {
var str string
for _, a := range aliases {
str += "[" + pterm.LightCyan(a.Alias) + "] "
}
return strings.Trim(str, " ")
}
func (c *Cache) AliasesByProjectString() string {
var str bytes.Buffer
w := new(tabwriter.Writer)
w.Init(&str, 10, 0, 0, ' ', tabwriter.AlignRight)
for _, p := range c.GetProjectsWithAliases() {
var pa string
pa += pterm.LightBlue("- ")
pa += fmt.Sprint(pterm.Bold.Sprint(p.String()) + " \t ")
pa += fmt.Sprint(ProjectAliasesString(c.GetProjectAliases(p)))
fmt.Fprintln(w, pa)
}
w.Flush()
return str.String()
}
func (c *Cache) ProjectString(p *gitlab.Project) string {
info := strings.Builder{}
@ -72,26 +39,7 @@ func (c *Cache) ProjectStrings(prefix string) []string {
projects = append(projects, p.NameWithNamespace)
}
}
return projects
}
func (c *Cache) AliasStrings(prefix string) []string {
aliases := make([]string, 0, len(c.Aliases))
for _, a := range c.Aliases {
if strings.HasPrefix(a.Alias, prefix) {
aliases = append(aliases, a.Alias)
}
}
return aliases
}
func (c *Cache) GetAliasByName(name string) *ProjectAlias {
for _, a := range c.Aliases {
if name == a.Alias {
return a
}
}
return nil
return slices.Clip(projects)
}
func (c *Cache) GetProjectByPath(path string) *gitlab.Project {
@ -103,6 +51,15 @@ func (c *Cache) GetProjectByPath(path string) *gitlab.Project {
return nil
}
func (c *Cache) GetProjectByRemoteAndId(remote string, id int) *gitlab.Project {
for _, p := range c.Projects {
if p.ID == id && p.Remote == remote {
return p
}
}
return nil
}
func (c *Cache) GetProjectByID(id int) *gitlab.Project {
for _, p := range c.Projects {
if p.ID == id {
@ -112,63 +69,15 @@ func (c *Cache) GetProjectByID(id int) *gitlab.Project {
return nil
}
func (c *Cache) GetProjectByAlias(alias *ProjectAlias) *gitlab.Project {
if alias == nil {
return nil
}
// Plural form of GetProjectByID
// Since multiple remotes may have the same project ID,
// this will return all matching
func (c *Cache) GetProjectsByID(id int) []*gitlab.Project {
projects := make([]*gitlab.Project, 0)
for _, p := range c.Projects {
if p.ID == alias.ProjectID {
return p
}
}
return nil
}
func (c *Cache) GetProjectAliases(project *gitlab.Project) []*ProjectAlias {
aliases := make([]*ProjectAlias, 0)
for _, alias := range c.Aliases {
if alias.ProjectID == project.ID {
aliases = append(aliases, alias)
}
}
return aliases
}
func (c *Cache) LoadProjects() {
progressInfo := c.gitlab.StreamProjects(c.config.Cache.Load.OwnerOnly)
c.Projects = make([]*gitlab.Project, 0)
pBar := pterm.DefaultProgressbar.
WithShowPercentage(true).
WithTotal(-1).
WithTitle("Listing GitLab Projects").
WithMaxWidth(100)
defer pBar.Stop()
for {
select {
case p := <-progressInfo.ProgressChan:
if pBar.Total == -1 {
pBar = pBar.WithTotal(p.TotalProjects)
pBar, _ = pBar.Start()
}
// This sucks, has to be a better way, and why is the logger incompatible
// with the progressbar?
pterm.Debug.Println(fmt.Sprintf("Update received: %#v", p))
pBar.Add(p.Projects)
case p := <-progressInfo.ProjectsChan:
c.Projects = append(c.Projects, p...)
case e := <-progressInfo.ErrorChan:
c.log.Error("Fetch GitLab projects error", c.log.Args("error", e))
case <-c.gitlab.Ctx.Done():
c.log.Warn("LoadProjects cancelled", c.log.Args("reason", c.gitlab.Ctx.Err()))
return
case <-progressInfo.DoneChan:
// pBar.Add(pBar.Total - curProjects)
fmt.Println("")
c.log.Info("Project load complete")
return
if p.ID == id {
projects = append(projects, p)
}
}
return projects
}

106
internal/projects/projects_alias.go Normal file
View File

@ -0,0 +1,106 @@
package projects
import (
"bytes"
"fmt"
"strings"
"text/tabwriter"
"github.com/pterm/pterm"
"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
"golang.org/x/exp/slices"
)
type ProjectAlias struct {
Alias string
ProjectID int
Remote string
}
func (c *Cache) GetProjectAliasStrings(project *gitlab.Project) []string {
aliases := c.GetProjectAliases(project)
strings := make([]string, len(aliases))
for i, a := range c.GetProjectAliases(project) {
strings[i] = a.Alias
}
return strings
}
func (c *Cache) GetProjectStringWithAliases(project *gitlab.Project) string {
aliases := c.GetProjectAliasStrings(project)
return fmt.Sprintf("%s (%s) -> %s",
project.Name,
strings.Join(aliases, ", "),
project.PathWithNamespace,
)
}
func ProjectAliasesString(aliases []*ProjectAlias) string {
var str string
for _, a := range aliases {
str += "[" + pterm.LightCyan(a.Alias) + "] "
}
return strings.Trim(str, " ")
}
func (c *Cache) AliasesByProjectString() string {
var str bytes.Buffer
w := new(tabwriter.Writer)
w.Init(&str, 10, 0, 0, ' ', tabwriter.AlignRight)
for _, p := range c.GetProjectsWithAliases() {
var pa string
pa += pterm.LightBlue("- ")
pa += fmt.Sprint(pterm.Bold.Sprint(p.String()) + " \t ")
pa += fmt.Sprint(ProjectAliasesString(c.GetProjectAliases(p)))
fmt.Fprintln(w, pa)
}
w.Flush()
return str.String()
}
func (c *Cache) AliasStrings(prefix string) []string {
aliases := make([]string, 0, len(c.Aliases))
for _, a := range c.Aliases {
if strings.HasPrefix(a.Alias, prefix) {
aliases = append(aliases, a.Alias)
}
}
return aliases
}
func (c *Cache) GetAliasByName(name string, gitlabs ...string) *ProjectAlias {
for _, a := range c.Aliases {
if len(gitlabs) > 0 && !slices.Contains(gitlabs, a.Remote) {
continue
}
if name == a.Alias {
return a
}
}
return nil
}
func (c *Cache) GetProjectByAlias(alias *ProjectAlias) *gitlab.Project {
if alias == nil {
return nil
}
for _, p := range c.Projects {
if p.ID == alias.ProjectID && p.Remote == alias.Remote {
return p
}
}
return nil
}
func (c *Cache) GetProjectAliases(project *gitlab.Project) []*ProjectAlias {
aliases := make([]*ProjectAlias, 0)
for _, alias := range c.Aliases {
if alias.ProjectID == project.ID {
aliases = append(aliases, alias)
}
}
return aliases
}