feat: fetch clones concurrently

Adds the `--parallel` (`-p`) flag, which will initialize/update the
clone cache in coroutines.

The default value is "one coroutine per CPU, with a maximum of 8".

cmd/root.go

@@ -69,12 +69,14 @@ func Execute() {
 
 func init() {
  rootCmd.PersistentFlags().BoolP("debug", "d", false, "Enable or disable debug mode")
+ rootCmd.PersistentFlags().BoolP("parallel", "p", true, "Fetch clones in parallel")
  rootCmd.PersistentFlags().
  StringP("gilt-dir", "c", "~/.gilt/clone", "Path to Gilt's clone dir")
  rootCmd.PersistentFlags().
  StringP("gilt-file", "f", "Giltfile.yaml", "Path to config file")
 
  _ = viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))
+ _ = viper.BindPFlag("parallel", rootCmd.PersistentFlags().Lookup("parallel"))
  _ = viper.BindPFlag("giltFile", rootCmd.PersistentFlags().Lookup("gilt-file"))
  _ = viper.BindPFlag("giltDir", rootCmd.PersistentFlags().Lookup("gilt-dir"))
  _ = viper.BindPFlag("repositories", rootCmd.PersistentFlags().Lookup("repositories"))

docs/docs/configuration.md

@@ -18,6 +18,7 @@ and/or directories to the desired destinations.
 ---
 giltDir: ~/.gilt/clone
 debug: false
+parallel: true
 repositories:
  - git: https://github.com/retr0h/ansible-etcd.git
  version: 77a95b7
@@ -55,6 +56,7 @@ The config file can be overriden/defined through env vars.
 GILT_GILTFILE=Giltfile.yaml \
 GILT_GILTDIR=~/.gilt/clone \
 GILT_DEBUG=false \
+GILT_PARALLEL=0 \
 gilt overlay
 ```
 
@@ -67,6 +69,7 @@ gilt \
  --gilt-file=Giltfile.yaml \
  --gilt-dir=~/.gilt/clone \
  --debug \
+ --parallel=false \
  overlay
 ```
 

examples/go-client/go.mod

@@ -15,10 +15,10 @@ require (
  github.com/gabriel-vasile/mimetype v1.4.3 // indirect
  github.com/go-playground/locales v0.14.1 // indirect
  github.com/go-playground/universal-translator v0.18.1 // indirect
- github.com/go-playground/validator/v10 v10.18.0 // indirect
+ github.com/go-playground/validator/v10 v10.19.0 // indirect
  github.com/leodido/go-urn v1.4.0 // indirect
  golang.org/x/crypto v0.19.0 // indirect
  golang.org/x/net v0.21.0 // indirect
- golang.org/x/sys v0.17.0 // indirect
+ golang.org/x/sys v0.18.0 // indirect
  golang.org/x/text v0.14.0 // indirect
 )

examples/go-client/go.sum

@@ -12,8 +12,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U=
-github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
+github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
@@ -22,15 +22,15 @@ github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
 github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/repositories/repositories.go

@@ -22,7 +22,9 @@ package repositories
 
 import (
  "log/slog"
+ "runtime"
  "strings"
+ "sync"
 
  "github.com/avfs/avfs"
 
@@ -31,6 +33,9 @@ import (
  "github.com/retr0h/gilt/v2/pkg/config"
 )
 
+// This should be a nice upper bound for parallel fetches
+const maxSlots = 8
+
 // New factory to create a new Repository instance.
 func New(
  appFs avfs.VFS,
@@ -73,7 +78,7 @@ func (r *Repositories) getCacheDir() (string, error) {
 
 // Overlay clone and extract the Repository items.
 func (r *Repositories) Overlay() error {
- if err := r.populateCloneCache(); err != nil {
+ if err := r.populateCloneCache(r.config.Parallel); err != nil {
  return err
  }
 
@@ -101,7 +106,7 @@ func (r *Repositories) Overlay() error {
 }
 
 // populateCloneCache ensure that all named repos exist and are up-to-date
-func (r *Repositories) populateCloneCache() error {
+func (r *Repositories) populateCloneCache(parallel bool) error {
  cacheDir, err := r.getCacheDir()
  if err != nil {
  r.logger.Error(
@@ -113,15 +118,65 @@ func (r *Repositories) populateCloneCache() error {
  return err
  }
 
- for _, c := range r.config.Repositories {
- if _, exists := r.cloneCache[c.Git]; exists {
- continue
- }
- targetDir, err := r.repoManager.Clone(c, cacheDir)
+ // Run all the clones concurrently (1 coroutine per CPU), up to 8 workers
+ slots := 1
+ if parallel {
+ slots = min(maxSlots, runtime.GOMAXPROCS(0))
+ }
+ var wg sync.WaitGroup
+ var mu sync.Mutex // Mutex to protect cloneCache
+ errChan := make(chan error, len(r.config.Repositories)) // Channel to collect errors
+ semaphore := make(chan struct{}, slots) // Semaphore to limit concurrency
+
+ for _, repo := range r.config.Repositories {
+ wg.Add(1)
+ go func(c config.Repository) {
+ defer wg.Done()
+ semaphore <- struct{}{}
+ defer func() { <-semaphore }()
+
+ if err := r.runPopulate(c, cacheDir, &mu); err != nil {
+ errChan <- err
+ }
+ }(repo)
+ }
+
+ // Roll up any errors fetching the above
+ wg.Wait()
+ close(errChan)
+ return r.anyErrors(errChan)
+}
+
+func (r *Repositories) runPopulate(c config.Repository, cacheDir string, mu *sync.Mutex) error {
+ mu.Lock()
+ if _, exists := r.cloneCache[c.Git]; exists {
+ mu.Unlock()
+ return nil
+ }
+ // Set a "stub" value to claim territory
+ // This worker is now responsible for populating the "full" value
+ r.cloneCache[c.Git] = ""
+ mu.Unlock()
+
+ // Initialize and/or update the clone (long-running operation outside the lock)
+ targetDir, err := r.repoManager.Clone(c, cacheDir)
+ if err != nil {
+ return err
+ }
+
+ // Rewrite with the "full" value
+ mu.Lock()
+ r.cloneCache[c.Git] = targetDir
+ mu.Unlock()
+
+ return nil
+}
+
+func (r *Repositories) anyErrors(errChan <-chan error) error {
+ for err := range errChan {
  if err != nil {
  return err
  }
- r.cloneCache[c.Git] = targetDir
  }
  return nil
 }

internal/repositories/repositories_public_test.go

@@ -61,6 +61,7 @@ func (suite *RepositoriesPublicTestSuite) NewTestRepositoriesManager(
 ) internal.RepositoriesManager {
  reposConfig := config.Repositories{
  Debug: false,
+ Parallel: true,
  GiltFile: "Giltfile.yaml",
  GiltDir: suite.giltDir,
  Repositories: repoConfig,

internal/repositories/repositories_test.go

@@ -133,7 +133,7 @@ func (suite *RepositoriesTestSuite) TestPopulateCloneCacheDedupesCloneCalls() {
  }
  // .Times(1) is the default behavior, but let's be explicit
  suite.mockRepo.EXPECT().Clone(gomock.Any(), gomock.Any()).Return(suite.giltDir, nil).Times(1)
- err := repos.populateCloneCache()
+ err := repos.populateCloneCache(false)
  assert.NoError(suite.T(), err)
 }
 

pkg/config/types.go

@@ -24,6 +24,8 @@ package config
 type Repositories struct {
  // Debug enable or disable debug option set from CLI.
  Debug bool `mapstruture:"debug"`
+ // Parallel enable or disable concurrent clone fetches.
+ Parallel bool ` mapstructure:"parallel"`
  // GiltFile path to Gilt's config file option set from CLI.
  GiltFile string ` mapstructure:"giltFile" validate:"required"`
  // GiltDir path to Gilt's clone dir option set from CLI.

pkg/repositories/repositories.go

@@ -164,6 +164,7 @@ func (r *Repositories) Overlay() error {
  slog.String("GiltDir", r.c.GiltDir),
  slog.String("GiltFile", r.c.GiltFile),
  slog.Bool("Debug", r.c.Debug),
+ slog.Bool("Parallel", r.c.Parallel),
  slog.Group("Repository", r.logRepositoriesGroup()...),
  )
 

test/integration/test_cli.bats

@@ -122,6 +122,20 @@ teardown() {
  echo "${output}" | grep -E ".*Preparing worktree.*HEAD is now at 77a95b7"
 }
 
+@test "invoke gilt overlay subcommand with parallelism disabled" {
+ run bash -c "cd ${GILT_TEST_BASE_TMP_DIR}; go run ${GILT_PROGRAM} --debug --parallel=false overlay"
+
+ [ "$status" -eq 0 ]
+ echo "${output}" | grep "Parallel=false"
+}
+
+@test "invoke gilt overlay subcommand with parallelism disabled (short)" {
+ run bash -c "cd ${GILT_TEST_BASE_TMP_DIR}; go run ${GILT_PROGRAM} -d -p=0 overlay"
+
+ [ "$status" -eq 0 ]
+ echo "${output}" | grep "Parallel=false"
+}
+
 @test "invoke gilt overlay when already cloned" {
  run bash -c "cd ${GILT_TEST_BASE_TMP_DIR}; go run ${GILT_PROGRAM} overlay"
  run bash -c "cd ${GILT_TEST_BASE_TMP_DIR}; go run ${GILT_PROGRAM} overlay"