fix: Refactor cloning and caching workflow (#71)

- Clone repositories into the cache with `--bare` to keep just the
  repo objects in cache, rather than an unused copy of the working
  tree.  This can yield a significant space savings in repositories
  with large numbers of files

- Clone repositories into the cache with `--filter=blob:none`.  This
  will skip downloading any objects from the repository in the
  initial copy of the clone, and only populate the repository
  metadata; subsequent operations will automatically download the
  required objects (and ONLY the required objects) from origin as
  needed.  This yields a massive space/performance boost.

- Make use of the `git worktree` subcommand to populate `dstDir`
  targets rather than `git checkout-index`.  This subcommand is
  designed for using a single repository as a source for multiple
  copies of the working tree, so it is a much better fit for what
  Gilt is trying to do.

- Remove the now-obsolete `CloneByTag`, `Reset`, and `CheckoutIndex`
  workflows.

With this change, the `tag:` and `sha:` directives in `Giltfile.yaml`
become 100% equivalent, with no difference in handling required.

Fixes: Issue #70

Co-authored-by: Nicolas Simonds <[email protected]>

internal/exec.go

@@ -23,4 +23,6 @@ package internal
 // ExecManager manager responsible for exec operations.
 type ExecManager interface {
  RunCmd(name string, args []string) error
+ RunCmdInDir(name string, args []string, cwd string) error
+ RunInTempDir(dir, pattern string, fn func(string) error) error
 }

internal/exec/exec.go

@@ -22,6 +22,7 @@ package exec
 
 import (
  "log/slog"
+ "os"
  "os/exec"
  "strings"
 )
@@ -37,26 +38,61 @@ func New(
  }
 }
 
+func (e *Exec) RunCmdImpl(
+ name string,
+ args []string,
+ cwd string,
+) error {
+ cmd := exec.Command(name, args...)
+ if cwd != "" {
+ cmd.Dir = cwd
+ }
+
+ commands := strings.Join(cmd.Args, " ")
+ e.logger.Debug("exec", slog.String("command", commands), slog.String("cwd", cwd))
+
+ out, err := cmd.CombinedOutput()
+ e.logger.Debug("result", slog.String("output", string(out)))
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
 // RunCmd execute the provided command with args.
 // Yeah, yeah, yeah, I know I cheated by using Exec in this package.
 func (e *Exec) RunCmd(
  name string,
  args []string,
 ) error {
- cmd := exec.Command(name, args...)
+ return e.RunCmdImpl(name, args, "")
+}
 
- if e.debug {
-  commands := strings.Join(cmd.Args, " ")
-  e.logger.Debug(
-  "exec",
- slog.String("command", commands),
-  )
- }
+func (e *Exec) RunCmdInDir(
+ name string,
+ args []string,
+ cwd string,
+) error {
+ return e.RunCmdImpl(name, args, cwd)
+}
 
- _, err := cmd.CombinedOutput()
+// RunInTempDir creates a temporary directory, and runs the provided function
+// with the name of the directory as input. Then it cleans up the temporary
+// directory.
+func (e *Exec) RunInTempDir(dir, pattern string, fn func(string) error) error {
+ tmpDir, err := os.MkdirTemp(dir, pattern)
  if err != nil {
  return err
  }
+ e.logger.Debug("created tempdir", slog.String("dir", tmpDir))
 
- return nil
+ // Ignoring errors as there's not much we can do and this is a cleanup function.
+ defer func() {
+ e.logger.Debug("removing tempdir", slog.String("dir", tmpDir))
+ _ = os.RemoveAll(tmpDir)
+ }()
+
+ // Run the provided function.
+ return fn(tmpDir)
 }

internal/exec/exec_public_test.go

@@ -71,6 +71,41 @@ func (suite *ExecManagerPublicTestSuite) TestRunCmdReturnsError() {
  assert.Contains(suite.T(), err.Error(), "not found")
 }
 
+func (suite *ExecManagerPublicTestSuite) TestRunCmdInDirOk() {
+ em := suite.NewTestExecManager(false)
+
+ err := em.RunCmdInDir("ls", []string{}, "/tmp")
+ assert.NoError(suite.T(), err)
+}
+
+func (suite *ExecManagerPublicTestSuite) TestRunCmdInDirWithDebug() {
+ suite.T().Skip("cannot seem to capture Stdout when logging in em")
+
+ em := suite.NewTestExecManager(true)
+
+ err := em.RunCmdInDir("echo", []string{"-n", "foo"}, "/tmp")
+ assert.NoError(suite.T(), err)
+}
+
+func (suite *ExecManagerPublicTestSuite) TestRunCmdInDirReturnsError() {
+ em := suite.NewTestExecManager(false)
+
+ err := em.RunCmdInDir("invalid", []string{"foo"}, "/tmp")
+ assert.Error(suite.T(), err)
+ assert.Contains(suite.T(), err.Error(), "not found")
+}
+
+func (suite *ExecManagerPublicTestSuite) TestRunInTempDirOk() {
+ em := suite.NewTestExecManager(false)
+
+ dir := ""
+ pattern := "test"
+ fn := func(string) error { return nil }
+
+ err := em.RunInTempDir(dir, pattern, fn)
+ assert.NoError(suite.T(), err)
+}
+
 // In order for `go test` to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run.
 func TestExecPublicTestSuite(t *testing.T) {

internal/git.go

@@ -23,7 +23,5 @@ package internal
 // GitManager manager responsible for Git operations.
 type GitManager interface {
  Clone(gitURL string, cloneDir string) error
- CloneByTag(gitURL string, gitTag string, cloneDir string) error
- Reset(cloneDir string, gitSHA string) error
- CheckoutIndex(dstDir string, cloneDir string) error
+ Worktree(cloneDir string, version string, dstDir string) error
 }

internal/git/git.go

@@ -48,38 +48,24 @@ func New(
  }
 }
 
-// Clone git clone repo.
+// Git clone repo. This is a bare repo, with only metadata to start with.
 func (g *Git) Clone(
  gitURL string,
  cloneDir string,
-) error {
- return g.execManager.RunCmd("git", []string{"clone", gitURL, cloneDir})
-}
-
-// CloneByTag git clone repo by tag.
-func (g *Git) CloneByTag(
- gitURL string,
- gitTag string,
- cloneDir string,
 ) error {
  return g.execManager.RunCmd(
  "git",
- []string{"clone", "--depth", "1", "--branch", gitTag, gitURL, cloneDir},
+ []string{"clone", "--bare", "--filter=blob:none", gitURL, cloneDir},
  )
 }
 
-// Reset to the given git version.
-func (g *Git) Reset(
+// Create a working tree from the repo in `cloneDir` at `version` in `dstDir`.
+// Under the covers, this will download any/all required objects from origin
+// into the cache
+func (g *Git) Worktree(
  cloneDir string,
- gitSHA string,
-) error {
- return g.execManager.RunCmd("git", []string{"-C", cloneDir, "reset", "--hard", gitSHA})
-}
-
-// CheckoutIndex checkout Repository.Git to Repository.DstDir.
-func (g *Git) CheckoutIndex(
+ version string,
  dstDir string,
- cloneDir string,
 ) error {
  dst, err := filepath.Abs(dstDir)
  if err != nil {
@@ -88,19 +74,20 @@ func (g *Git) CheckoutIndex(
 
  g.logger.Info(
  "extracting",
+ slog.String("from", cloneDir),
+ slog.String("version", version),
  slog.String("to", dst),
  )
 
- cmdArgs := []string{
- "-C",
+ err = g.execManager.RunCmdInDir(
+ "git",
+ []string{"worktree", "add", "--force", dst, version},
  cloneDir,
- "checkout-index",
- "--force",
- "--all",
- "--prefix",
- // Trailing separator needed by git checkout-index.
- dst + string(os.PathSeparator),
+ )
+ // `git worktree add` creates a breadcrumb file back to the original repo;
+ // this is just junk data in our use case, so get rid of it
+ if err == nil {
+ _ = os.Remove(filepath.Join(dst, ".git"))
  }
-
- return g.execManager.RunCmd("git", cmdArgs)
+ return err
 }

internal/git/git_public_test.go

@@ -76,7 +76,7 @@ func (suite *GitManagerPublicTestSuite) SetupTest() {
 
 func (suite *GitManagerPublicTestSuite) TestCloneOk() {
  suite.mockExec.EXPECT().
- RunCmd("git", []string{"clone", suite.gitURL, suite.cloneDir}).
+ RunCmd("git", []string{"clone", "--bare", "--filter=blob:none", suite.gitURL, suite.cloneDir}).
  Return(nil)
 
  err := suite.gm.Clone(suite.gitURL, suite.cloneDir)
@@ -91,63 +91,6 @@ func (suite *GitManagerPublicTestSuite) TestCloneReturnsError() {
  assert.Error(suite.T(), err)
 }
 
-func (suite *GitManagerPublicTestSuite) TestCloneByTagOk() {
- suite.mockExec.EXPECT().
- RunCmd("git", []string{"clone", "--depth", "1", "--branch", suite.gitTag, suite.gitURL, suite.cloneDir}).
- Return(nil)
-
- err := suite.gm.CloneByTag(suite.gitURL, suite.gitTag, suite.cloneDir)
- assert.NoError(suite.T(), err)
-}
-
-func (suite *GitManagerPublicTestSuite) TestCloneByTagReturnsError() {
- errors := errors.New("tests error")
- suite.mockExec.EXPECT().RunCmd(gomock.Any(), gomock.Any()).Return(errors)
-
- err := suite.gm.CloneByTag(suite.gitURL, suite.gitTag, suite.cloneDir)
- assert.Error(suite.T(), err)
-}
-
-func (suite *GitManagerPublicTestSuite) TestResetOk() {
- suite.mockExec.EXPECT().
- RunCmd("git", []string{"-C", suite.cloneDir, "reset", "--hard", suite.gitSHA})
-
- err := suite.gm.Reset(suite.cloneDir, suite.gitSHA)
- assert.NoError(suite.T(), err)
-}
-
-func (suite *GitManagerPublicTestSuite) TestResetReturnsError() {
- errors := errors.New("tests error")
- suite.mockExec.EXPECT().RunCmd(gomock.Any(), gomock.Any()).Return(errors)
-
- err := suite.gm.Reset(suite.cloneDir, suite.gitSHA)
- assert.Error(suite.T(), err)
-}
-
-func (suite *GitManagerPublicTestSuite) TestCheckoutIndexOk() {
- cmdArgs := []string{
- "-C",
- suite.cloneDir,
- "checkout-index",
- "--force",
- "--all",
- "--prefix",
- suite.dstDir + string(os.PathSeparator),
- }
- suite.mockExec.EXPECT().RunCmd("git", cmdArgs).Return(nil)
-
- err := suite.gm.CheckoutIndex(suite.dstDir, suite.cloneDir)
- assert.NoError(suite.T(), err)
-}
-
-func (suite *GitManagerPublicTestSuite) TestCheckoutIndexReturnsError() {
- errors := errors.New("tests error")
- suite.mockExec.EXPECT().RunCmd(gomock.Any(), gomock.Any()).Return(errors)
-
- err := suite.gm.CheckoutIndex(suite.dstDir, suite.cloneDir)
- assert.Error(suite.T(), err)
-}
-
 // In order for `go test` to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run.
 func TestGitManagerPublicTestSuite(t *testing.T) {