wip

internal/config/config.go

@@ -11,6 +11,9 @@ type Config struct {
 }
 
 type cacheConfig struct {
-	Ttl  time.Duration `yaml:"ttl" json:"ttl"`
-	File string
+	Ttl   time.Duration `yaml:"ttl" json:"ttl"`
+	File  string        `yaml:"file" json:"file"`
+	Clear struct {
+		ClearAliases bool `yaml:"clearAliases" json:"clearAliases"`
+	} `yaml:"clear" json:"clear"`
 }

internal/gitlab/gitlab.go

@@ -1,31 +1,154 @@
 package gitlab
 
-import "github.com/xanzy/go-gitlab"
+import (
+	"context"
+	"time"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+const defProjectsPerPage = 100
 
 type Client struct {
+	Ctx    context.Context
 	gitlab *gitlab.Client
 }
 
-type ProjectInfo struct {
-	Name        string `yaml:"name" json:"name"`
-	Id          int    `yaml:"id" json:"id"`
-	Path        string `yaml:"path" json:"path"`
-	URI         string `yaml:"uri" json:"uri"`
-	Description string `yaml:"description" json:"description"`
+type Project struct {
+	ID            int
+	Description   string
+	SSHURLToRepo  string
+	HTTPURLToRepo string
+	WebURL        string
+	// Owner             User
+	Name              string
+	NameWithNamespace string
+	Path              string
+	PathWithNamespace string
+	AvatarURL         string
+	LastActivityAt    time.Time
 }
 
-type ProjectAlias struct {
-	Alias     string
-	ProjectID string
-	Project   *ProjectInfo
+type User struct {
+	ID        int
+	Username  string
+	Email     string
+	Name      string
+	AvatarURL string
 }
 
-func NewGitlabClient(host, token string) (*Client, error) {
+type ProgressInfo struct {
+	ProgressChan chan Progress
+	ProjectsChan chan []*Project
+	ErrorChan    chan error
+	DoneChan     chan interface{}
+}
+
+type Progress struct {
+	Page          int
+	Pages         int
+	Projects      int
+	TotalProjects int
+}
+
+// Given there may be thousands of projects, this will return
+// channels that stream progress info and then finally the full
+// list of projects on separate channels
+func (c *Client) StreamProjects() *ProgressInfo {
+	pi := &ProgressInfo{
+		ProgressChan: make(chan Progress),
+		ProjectsChan: make(chan []*Project),
+		ErrorChan:    make(chan error),
+		DoneChan:     make(chan interface{}),
+	}
+
+	go c.streamProjects(pi)
+
+	return pi
+}
+
+func (c *Client) streamProjects(pi *ProgressInfo) {
+	defer close(pi.ProgressChan)
+	defer close(pi.ProjectsChan)
+
+	listOpts := &gitlab.ListProjectsOptions{
+		ListOptions: gitlab.ListOptions{
+			PerPage: defProjectsPerPage,
+			Page:    1,
+		},
+		Archived: new(bool),
+	}
+
+	var numProjects int
+	for {
+		projects, resp, err := c.ListProjects(listOpts)
+
+		// We're done when we have it all or our context is done
+		if c.Ctx.Err() != nil || resp.NextPage == 0 {
+			pi.DoneChan <- nil
+			break
+		}
+
+		if err != nil {
+			pi.ErrorChan <- err
+		}
+
+		numProjects += len(projects)
+		pi.ProjectsChan <- projects
+		pi.ProgressChan <- Progress{
+			Page:          resp.CurrentPage,
+			Pages:         resp.TotalPages,
+			Projects:      numProjects,
+			TotalProjects: resp.TotalItems,
+		}
+		listOpts.Page = resp.NextPage
+	}
+}
+
+// Returns a list of projects along with the next page and an error
+// if there was an error
+func (c *Client) ListProjects(opts *gitlab.ListProjectsOptions) (
+	[]*Project, *gitlab.Response, error) {
+	pList := make([]*Project, 0)
+	projects, resp, err := c.gitlab.Projects.ListProjects(
+		opts,
+		gitlab.WithContext(c.Ctx),
+	)
+	if err == nil {
+		pList = append(pList, c.handleProjects(projects)...)
+	}
+	return pList, resp, err
+}
+
+func (c *Client) handleProjects(projects []*gitlab.Project) []*Project {
+	// Opportunity to perform any filtering or additional lookups
+	// on a per-project basis
+	pList := make([]*Project, 0, len(projects))
+	for _, project := range projects {
+		p := &Project{
+			ID:                project.ID,
+			Description:       project.Description,
+			SSHURLToRepo:      project.SSHURLToRepo,
+			HTTPURLToRepo:     project.HTTPURLToRepo,
+			WebURL:            project.WebURL,
+			Name:              project.Name,
+			NameWithNamespace: project.NameWithNamespace,
+			Path:              project.Path,
+			PathWithNamespace: project.PathWithNamespace,
+			LastActivityAt:    *project.LastActivityAt,
+		}
+		pList = append(pList, p)
+	}
+	return pList
+}
+
+func NewGitlabClient(ctx context.Context, host, token string) (*Client, error) {
 	client, err := gitlab.NewClient(token, gitlab.WithBaseURL(host))
 	if err != nil {
 		return nil, err
 	}
 	gitlabClient := &Client{
+		Ctx:    ctx,
 		gitlab: client,
 	}
 	return gitlabClient, nil

internal/projects/cache.go

@@ -7,30 +7,33 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pterm/pterm"
 	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
-	"golang.org/x/exp/slog"
 	"gopkg.in/yaml.v3"
 )
 
 type Cache struct {
-	Projects     []*gitlab.ProjectInfo
-	Aliases      []*gitlab.ProjectAlias
+	Projects     []*gitlab.Project
+	Aliases      []*ProjectAlias
 	Updated      time.Time
 	readFromFile bool
 	lock         *sync.Mutex
 	ttl          time.Duration
 	file         string
-	log          *slog.Logger
+	log          *pterm.Logger
+	gitlab       *gitlab.Client
 }
 
 type CacheOpts struct {
 	Path   string
 	TTL    time.Duration
-	Logger *slog.Logger
+	Logger *pterm.Logger
+	Gitlab *gitlab.Client
 }
 
-// Load cache, if already loaded and
-// up to date,
+// Load cache, if already loaded and up to date, nothing is done.
+// If the cache is not yet loaded from disk, it is loaded
+// If the updated timestamp is beyond the set ttl, a refresh is triggered
 func (c *Cache) Load() error {
 	var err error
 	if !c.readFromFile {
@@ -43,27 +46,34 @@ func (c *Cache) Load() error {
 	return err
 }
 
-func (c *Cache) Write() {
+// Saves the current state of the cache to disk
+func (c *Cache) write() {
 	file, err := os.OpenFile(c.file, os.O_RDWR, fs.ModeAppend)
 	if err != nil {
-		c.log.Error("Failed to write cache to disk", "error", err)
+		c.log.Error("Failed to write cache to disk", c.log.Args("error", err))
 	}
 	d := yaml.NewEncoder(file)
 	if err = d.Encode(*c); err != nil {
-		c.log.Error("Failed to Marshal cache to yaml", "error", err)
+		c.log.Error("Failed to Marshal cache to yaml", c.log.Args("error", err))
 	} else {
-		c.log.Info("Cache saved to disk")
+		c.log.Debug("Cache saved to disk")
 	}
 }
+func (c *Cache) Write() {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.write()
+}
 
+// Loads and unmarshals the project cache from disk.
 func (c *Cache) Read() {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	c.log.Info("Reading project cache from disk", "file", c.file)
+	c.log.Debug("Reading project cache from disk", c.log.Args("file", c.file))
 
 	file, err := os.Open(c.file)
 	if err != nil {
-		c.log.Error("Failed to read project cache", "error", err)
+		c.log.Error("Failed to read project cache", c.log.Args("error", err))
 		return
 	}
 
@@ -78,11 +88,32 @@ func (c *Cache) Read() {
 	c.log.Debug(c.String())
 }
 
+// Resets projects cache and also optionally clears
+// project aliase cache. Writes to disk.
+func (c *Cache) clear(clearAliases bool) {
+	c.log.Info("Clearing project cache")
+	c.Projects = make([]*gitlab.Project, 0)
+	if clearAliases {
+		c.log.Info("Clearing project alias cache")
+		c.Aliases = make([]*ProjectAlias, 0)
+	}
+	c.setUpdated()
+	c.log.Debug(c.String())
+}
+func (c *Cache) Clear(clearAliases bool) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.clear(clearAliases)
+}
+
 func (c *Cache) refresh() {
 	c.log.Info("Refreshing project cache, this may take a while")
 	defer c.setUpdated()
+	c.LoadProjects()
 }
 
+// Iterates through all GitLab projects the user has access to, updating
+// the project cache where necessary
 func (c *Cache) Refresh() {
 	c.lock.Lock()
 	defer c.lock.Unlock()
@@ -121,12 +152,13 @@ func NewProjectCache(opts *CacheOpts) (*Cache, error) {
 	}
 
 	cache := &Cache{
-		Projects: make([]*gitlab.ProjectInfo, 0),
-		Aliases:  make([]*gitlab.ProjectAlias, 0),
+		Projects: make([]*gitlab.Project, 0),
+		Aliases:  make([]*ProjectAlias, 0),
 		file:     opts.Path,
 		ttl:      opts.TTL,
 		lock:     &sync.Mutex{},
 		log:      opts.Logger,
+		gitlab:   opts.Gitlab,
 	}
 
 	return cache, err

internal/projects/projects.go

@@ -0,0 +1,53 @@
+package projects
+
+import (
+	"fmt"
+
+	"github.com/pterm/pterm"
+	"gitlab.sweetwater.com/it/devops/tools/gitlab-project-manager/internal/gitlab"
+)
+
+type ProjectAlias struct {
+	Alias     string
+	ProjectID string
+	Project   *gitlab.Project
+}
+
+func (c *Cache) LoadProjects() {
+	progressInfo := c.gitlab.StreamProjects()
+	c.Projects = make([]*gitlab.Project, 0)
+
+	pBar := pterm.DefaultProgressbar.
+		WithShowPercentage(true).
+		WithTotal(-1).
+		WithTitle("Listing GitLab Projects").
+		WithMaxWidth(0)
+
+	defer pBar.Stop()
+
+	var curProjects int
+	for {
+		select {
+		case p := <-progressInfo.ProgressChan:
+			if pBar.Total == -1 {
+				pBar = pBar.WithTotal(p.TotalProjects)
+				pBar, _ = pBar.Start()
+			}
+			// This sucks, has to be a better way, and why is the logger incompatible
+			// with the progressbar?
+			pterm.Debug.Println(fmt.Sprintf("Update received: %#v", p))
+			pBar.Add(p.Projects - curProjects)
+			curProjects = p.Projects
+		case p := <-progressInfo.ProjectsChan:
+			c.Projects = append(c.Projects, p...)
+		case e := <-progressInfo.ErrorChan:
+			c.log.Error("Fetch GitLab projects error", c.log.Args("error", e))
+		case <-c.gitlab.Ctx.Done():
+			c.log.Warn("LoadProjects cancelled", c.log.Args("reason", c.gitlab.Ctx.Err()))
+			return
+		case <-progressInfo.DoneChan:
+			c.log.Info("Project load complete")
+			return
+		}
+	}
+}