From e187d543a85803e6300e11870be606fc769055ec Mon Sep 17 00:00:00 2001 From: John Dewey Date: Wed, 15 Aug 2018 12:38:13 -0700 Subject: [PATCH] Ability to copy files out of the source repo With the following gilt.yml, Gilt can copy the src to the dst. - git: https://github.com/lorin/openstack-ansible-modules.git version: 2677cc3 sources: - src: "*_manage" dstDir: library - src: nova_quota dstDir: library - src: neutron_router dstFile: library/neutron_router.py - src: tests dstDir: tests Reworked some of the previous schema as well to be more consistent existing naming. --- .gitignore | 1 + Makefile | 19 ++- README.md | 20 ++- copy/copy.go | 145 ++++++++++++++++++ git/git.go | 17 +- git/git_public_test.go | 6 +- git/git_test.go | 4 +- main.go | 1 - repositories/repositories.go | 17 +- .../repositories_integration_test.go | 77 +++++----- repositories/repositories_public_test.go | 75 ++++++--- repositories/repositories_test.go | 145 ++++++++++++++++-- repositories/schema.go | 47 ++++-- repository/repository.go | 78 ++++++++-- repository/repository_public_test.go | 4 +- test/gilt.yml | 16 +- test/integration/test_cli.bats | 100 ++++++++++-- test/resources/copy/dir/foo | 0 test/resources/copy/file | 0 util/util.go | 28 ++++ util/util_public_test.go | 38 +++++ 21 files changed, 704 insertions(+), 134 deletions(-) create mode 100644 copy/copy.go rename git/schema.go => repositories/repositories_integration_test.go (55%) create mode 100644 test/resources/copy/dir/foo create mode 100644 test/resources/copy/file diff --git a/.gitignore b/.gitignore index 9593fc0..c55e4cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .build/ coverage* +test/integration/tmp/ diff --git a/Makefile b/Makefile index 19c07d0..63ca229 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ VENDOR := vendor PKGS := $(shell go list ./... | grep -v /$(VENDOR)/) +SRC = $(shell find . -type f -name '*.go' -not -path "./$(VENDOR)/*") $(if $(PKGS), , $(error "go list failed")) PKGS_DELIM := $(shell echo $(PKGS) | sed -e 's/ /,/g') GITCOMMIT ?= $(shell git rev-parse --short HEAD) @@ -19,9 +20,9 @@ LDFLAGS := -s \ BUILDDIR := .build export GOCACHE = off -test: fmt lint vet bats +test: fmtcheck lint vet bats @echo "+ $@" - @go test -parallel 5 -covermode=count ./... + @go test -tags=integration -parallel 5 -covermode=count ./... bats: @echo "+ $@" @@ -31,22 +32,26 @@ cover: @echo "+ $@" $(shell [ -e coverage.out ] && rm coverage.out) @echo "mode: count" > coverage-all.out - $(foreach pkg,$(PKGS),\ - go test -coverprofile=coverage.out -covermode=count $(pkg);\ + @$(foreach pkg,$(PKGS),\ + go test -tags=integration -coverprofile=coverage.out -covermode=count $(pkg);\ tail -n +2 coverage.out >> coverage-all.out;) @go tool cover -html=coverage-all.out -o=coverage-all.html fmt: @echo "+ $@" - @gofmt -s -l . | grep -v $(VENDOR) | tee /dev/stderr + @gofmt -s -l -w $(SRC) + +fmtcheck: + @echo "+ $@" + @bash -c "diff -u <(echo -n) <(gofmt -d $(SRC))" lint: @echo "+ $@" - @echo $(PKGS) | xargs -L1 golint -set_exit_status | tee /dev/stderr + @echo $(PKGS) | xargs -L1 golint -set_exit_status vet: @echo "+ $@" - @go vet $(shell go list ./... | grep -v $(VENDOR)) + @go vet $(PKGS) clean: @echo "+ $@" diff --git a/README.md b/README.md index e613d46..b4730dd 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,26 @@ not 100% compatible with the python version, and not yet complete. Create the giltfile (`gilt.yml`). -Clone the specified `url`@`version` to the configurable path `--giltdir`, and -extract the repository to the provided `dst`. +Clone the specified `url`@`version` to the configurable path `--giltdir`. +Extract the repo the `dstDir` when `dstDir` is provided. Otherwise, copy files +and/or directories to the desired destinations. ```yaml --- -- url: https://github.com/retr0h/ansible-etcd.git +- git: https://github.com/retr0h/ansible-etcd.git version: 77a95b7 - dst: roles/retr0h.ansible-etcd + dstDir: roles/retr0h.ansible-etcd +- git: https://github.com/lorin/openstack-ansible-modules.git + version: 2677cc3 + sources: + - src: "*_manage" + dstDir: library + - src: nova_quota + dstDir: library + - src: neutron_router + dstFile: library/neutron_router.py + - src: tests + dstDir: tests ``` Overlay a remote repository into the destination provided. diff --git a/copy/copy.go b/copy/copy.go new file mode 100644 index 0000000..b5c859c --- /dev/null +++ b/copy/copy.go @@ -0,0 +1,145 @@ +// Copyright (c) 2018 John Dewey + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +// Package copy taken from terraform, and added as copy package to avoid testing +// directly and dropping coverage of util package. +// https://github.com/hashicorp/terraform/blob/master/helper/copy/copy.go +package copy + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// From: https://gist.github.com/m4ng0squ4sh/92462b38df26839a3ca324697c8cba04 + +// File copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. The file mode will be copied from the source and +// the copied data is synced/flushed to stable storage. +func File(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + +// Dir recursively copies a directory tree, attempting to preserve permissions. +// Source directory must exist, destination directory must *not* exist. +// Symlinks are ignored and skipped. +func Dir(src string, dst string) (err error) { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf("source is not a directory") + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf("destination already exists") + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := ioutil.ReadDir(src) + if err != nil { + return + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + // If the entry is a symlink, we copy the contents + for entry.Mode()&os.ModeSymlink != 0 { + + target, err := filepath.EvalSymlinks(srcPath) + if err != nil { + return err + } + + entry, err = os.Stat(target) + if err != nil { + return err + } + } + + if entry.IsDir() { + err = Dir(srcPath, dstPath) + if err != nil { + return + } + } else { + err = File(srcPath, dstPath) + if err != nil { + return + } + } + } + + return +} diff --git a/git/git.go b/git/git.go index c875a26..b55a677 100644 --- a/git/git.go +++ b/git/git.go @@ -47,15 +47,15 @@ func NewGit(debug bool) *Git { } } -// Clone clone Repository.URL to Repository.getCloneDir, and hard checkout +// Clone clone Repository.Git to Repository.getCloneDir, and hard checkout // to Repository.Version. func (g *Git) Clone(repository repository.Repository) error { cloneDir := repository.GetCloneDir() - msg := fmt.Sprintf("[%s@%s]:", aurora.Magenta(repository.URL), aurora.Magenta(repository.Version)) + msg := fmt.Sprintf("[%s@%s]:", aurora.Magenta(repository.Git), aurora.Magenta(repository.Version)) fmt.Println(msg) - msg = fmt.Sprintf("%-2s - %s '%s'", "", aurora.Cyan("Cloning to"), aurora.Cyan(cloneDir)) + msg = fmt.Sprintf("%-2s - Cloning to '%s'", "", aurora.Cyan(cloneDir)) fmt.Println(msg) if _, err := os.Stat(cloneDir); os.IsNotExist(err) { @@ -67,7 +67,8 @@ func (g *Git) Clone(repository repository.Repository) error { return err } } else { - msg := fmt.Sprintf("%-4s * %s", "", aurora.Brown("Clone already exists")) + bang := aurora.Bold(aurora.Red("!")) + msg := fmt.Sprintf("%-2s %s %s", "", bang, aurora.Brown("Clone already exists")) fmt.Println(msg) } @@ -76,7 +77,7 @@ func (g *Git) Clone(repository repository.Repository) error { func (g *Git) clone(repository repository.Repository) error { cloneDir := repository.GetCloneDir() - err := RunCommand(g.Debug, "git", "clone", repository.URL, cloneDir) + err := RunCommand(g.Debug, "git", "clone", repository.Git, cloneDir) return err } @@ -88,15 +89,15 @@ func (g *Git) reset(repository repository.Repository) error { return err } -// CheckoutIndex checkout Repository.Git to Repository.Dst. +// CheckoutIndex checkout Repository.Git to Repository.DstDir. func (g *Git) CheckoutIndex(repository repository.Repository) error { cloneDir := repository.GetCloneDir() - dstDir, err := FilePathAbs(repository.Dst) + dstDir, err := FilePathAbs(repository.DstDir) if err != nil { return err } - msg := fmt.Sprintf("%-2s - %s '%s'", "", aurora.Cyan("Extracting to"), aurora.Cyan(dstDir)) + msg := fmt.Sprintf("%-2s - Extracting to '%s'", "", aurora.Cyan(dstDir)) fmt.Println(msg) cmdArgs := []string{ diff --git a/git/git_public_test.go b/git/git_public_test.go index 76cf7d5..025e3b3 100644 --- a/git/git_public_test.go +++ b/git/git_public_test.go @@ -43,9 +43,9 @@ type GitTestSuite struct { func (suite *GitTestSuite) SetupTest() { suite.g = git.NewGit(false) suite.r = repository.Repository{ - URL: "https://example.com/user/repo.git", + Git: "https://example.com/user/repo.git", Version: "abc1234", - Dst: "path/user.repo", + DstDir: "path/user.repo", GiltDir: testutil.CreateTempDirectory(), } } @@ -136,7 +136,7 @@ func (suite *GitTestSuite) TestCheckoutIndex() { return err } - dstDir, _ := git.FilePathAbs(suite.r.Dst) + dstDir, _ := git.FilePathAbs(suite.r.DstDir) got := git.MockRunCommand(anon) want := []string{ fmt.Sprintf("git -C %s/https---example.com-user-repo.git-abc1234 checkout-index --force --all --prefix %s", diff --git a/git/git_test.go b/git/git_test.go index 857daea..15e0a86 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -39,9 +39,9 @@ type GitTestSuite struct { func (suite *GitTestSuite) SetupTest() { suite.g = NewGit(false) suite.r = repository.Repository{ - URL: "https://example.com/user/repo.git", + Git: "https://example.com/user/repo.git", Version: "abc1234", - Dst: "path/user.repo", + DstDir: "path/user.repo", GiltDir: testutil.CreateTempDirectory(), } } diff --git a/main.go b/main.go index 0e8c459..80f9058 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -// Package main TODO package main import ( diff --git a/repositories/repositories.go b/repositories/repositories.go index 3239049..4c811ae 100644 --- a/repositories/repositories.go +++ b/repositories/repositories.go @@ -25,6 +25,7 @@ import ( "errors" "fmt" "io/ioutil" + "os" "strings" "github.com/ghodss/yaml" @@ -117,12 +118,24 @@ func (r *Repositories) Overlay() error { return err } - // Checkout into repository.Dst. - if repository.Dst != "" { + // Checkout into repository.DstDir. + if repository.DstDir != "" { + // Delete dstDir since Checkout-Index does not clean old files that may + // no longer exist in repository. + if info, err := os.Stat(repository.DstDir); err == nil && info.Mode().IsDir() { + os.RemoveAll(repository.DstDir) + } if err := g.CheckoutIndex(repository); err != nil { return err } } + + // Copy sources from Repository.Src to Repository.DstDir or Repository.DstFile. + if len(repository.Sources) > 0 { + if err := repository.CopySources(); err != nil { + return err + } + } } return nil diff --git a/git/schema.go b/repositories/repositories_integration_test.go similarity index 55% rename from git/schema.go rename to repositories/repositories_integration_test.go index 039e470..74a3e13 100644 --- a/git/schema.go +++ b/repositories/repositories_integration_test.go @@ -1,3 +1,4 @@ +// +build integration // Copyright (c) 2018 John Dewey // Permission is hereby granted, free of charge, to any person obtaining a copy @@ -18,47 +19,41 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -package git +package repositories_test -const configSchema = ` -{ - "type": "array", - "$schema": "http://json-schema.org/draft-07/schema#", - "minItems": 1, - "items": { - "type": "object", - "properties": { - "git": { - "type": "string" - }, - "sha": { - "type": "string", - "pattern": "^[0-9a-f]{5,40}$" - }, - "dst": { - "type": "string" - }, - "files": { - "type": "array" - } - }, - "additionalProperties": true, - "oneOf": [ - { - "required": [ - "git", - "sha", - "dst" - ] - }, - { - "required": [ - "git", - "sha", - "files" - ] - } - ] - } +import ( + "fmt" + + "github.com/retr0h/go-gilt/test/testutil" + "github.com/stretchr/testify/assert" +) + +func (suite *RepositoriesTestSuite) TestOverlayRemovesSrcDirPriorToCheckoutIndex() { + tempDir := testutil.CreateTempDirectory() + data := fmt.Sprintf(` +--- +- git: https://github.com/retr0h/ansible-etcd.git + version: 77a95b7 + dstDir: %s/retr0h.ansible-etcd +`, tempDir) + suite.r.UnmarshalYAML([]byte(data)) + suite.r.Overlay() + err := suite.r.Overlay() + + assert.NoError(suite.T(), err) } + +func (suite *RepositoriesTestSuite) TestOverlayFailsCopySourcesReturnsError() { + data := ` +--- +- git: https://github.com/lorin/openstack-ansible-modules.git + version: 2677cc3 + sources: + - src: "*_manage" + dstDir: /super/invalid/path/to/write/to ` + suite.r.UnmarshalYAML([]byte(data)) + err := suite.r.Overlay() + + assert.Error(suite.T(), err) +} diff --git a/repositories/repositories_public_test.go b/repositories/repositories_public_test.go index cc41553..f937a8c 100644 --- a/repositories/repositories_public_test.go +++ b/repositories/repositories_public_test.go @@ -74,16 +74,33 @@ foo: bar func (suite *RepositoriesTestSuite) TestUnmarshalYAML() { data := ` --- -- url: https://example.com/user/repo.git +- git: https://example.com/user/repo.git version: abc1234 - dst: path/user.repo + dstDir: path/user.repo + +- git: https://example.com/user/repo.git + version: abc6789 + sources: + - src: foo + dstFile: bar ` err := suite.r.UnmarshalYAML([]byte(data)) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "https://example.com/user/repo.git", suite.r.Items[0].URL) - assert.Equal(suite.T(), "abc1234", suite.r.Items[0].Version) - assert.Equal(suite.T(), "path/user.repo", suite.r.Items[0].Dst) + + firstItem := suite.r.Items[0] + assert.Equal(suite.T(), "https://example.com/user/repo.git", firstItem.Git) + assert.Equal(suite.T(), "abc1234", firstItem.Version) + assert.Equal(suite.T(), "path/user.repo", firstItem.DstDir) + assert.Empty(suite.T(), firstItem.Sources) + + secondItem := suite.r.Items[1] + fmt.Println(secondItem) + assert.Equal(suite.T(), "https://example.com/user/repo.git", secondItem.Git) + assert.Equal(suite.T(), "abc6789", secondItem.Version) + assert.Empty(suite.T(), secondItem.DstDir) + assert.Equal(suite.T(), "foo", secondItem.Sources[0].Src) + assert.Equal(suite.T(), "bar", secondItem.Sources[0].DstFile) } func (suite *RepositoriesTestSuite) TestUnmarshalYAMLFileReturnsErrorWithMissingFile() { @@ -99,17 +116,30 @@ func (suite *RepositoriesTestSuite) TestUnmarshalYAMLFile() { suite.r.Filename = path.Join("..", "test", "gilt.yml") suite.r.UnmarshalYAMLFile() - assert.NotNil(suite.T(), suite.r.Items[0].URL) - assert.NotNil(suite.T(), suite.r.Items[0].Version) - assert.NotNil(suite.T(), suite.r.Items[0].Dst) + firstItem := suite.r.Items[0] + assert.NotNil(suite.T(), firstItem.Git) + assert.NotNil(suite.T(), firstItem.Version) + assert.NotNil(suite.T(), firstItem.DstDir) + assert.Empty(suite.T(), firstItem.Sources) + + secondItem := suite.r.Items[1] + assert.NotNil(suite.T(), secondItem.Git) + assert.NotNil(suite.T(), secondItem.Version) + assert.Empty(suite.T(), secondItem.DstDir) + assert.NotNil(suite.T(), secondItem.Sources[0].Src) + assert.NotNil(suite.T(), secondItem.Sources[0].DstFile) + assert.NotNil(suite.T(), secondItem.Sources[1].Src) + assert.NotNil(suite.T(), secondItem.Sources[1].DstFile) + assert.NotNil(suite.T(), secondItem.Sources[2].Src) + assert.NotNil(suite.T(), secondItem.Sources[2].DstFile) } func (suite *RepositoriesTestSuite) TestOverlayFailsCloneReturnsError() { data := ` --- -- url: invalid. +- git: invalid. version: abc1234 - dst: path/user.repo + dstDir: path/user.repo ` suite.r.UnmarshalYAML([]byte(data)) anon := func() error { @@ -125,9 +155,9 @@ func (suite *RepositoriesTestSuite) TestOverlayFailsCloneReturnsError() { func (suite *RepositoriesTestSuite) TestOverlayFailsCheckoutIndexReturnsError() { data := ` --- -- url: https://example.com/user/repo.git +- git: https://example.com/user/repo.git version: abc1234 - dst: /invalid/directory + dstDir: /invalid/directory ` suite.r.UnmarshalYAML([]byte(data)) anon := func() error { @@ -143,9 +173,14 @@ func (suite *RepositoriesTestSuite) TestOverlayFailsCheckoutIndexReturnsError() func (suite *RepositoriesTestSuite) TestOverlay() { data := ` --- -- url: https://example.com/user/repo.git +- git: https://example.com/user/repo1.git + version: abc1234 + dstDir: path/user.repo +- git: https://example.com/user/repo2.git version: abc1234 - dst: path/user.repo + sources: + - src: foo + dstFile: bar ` suite.r.UnmarshalYAML([]byte(data)) anon := func() error { @@ -155,15 +190,19 @@ func (suite *RepositoriesTestSuite) TestOverlay() { return err } - dstDir, _ := git.FilePathAbs(suite.r.Items[0].Dst) + dstDir, _ := git.FilePathAbs(suite.r.Items[0].DstDir) got := git.MockRunCommand(anon) want := []string{ - fmt.Sprintf("git clone https://example.com/user/repo.git %s/https---example.com-user-repo.git-abc1234", + fmt.Sprintf("git clone https://example.com/user/repo1.git %s/https---example.com-user-repo1.git-abc1234", repositories.GiltDir), - fmt.Sprintf("git -C %s/https---example.com-user-repo.git-abc1234 reset --hard abc1234", + fmt.Sprintf("git -C %s/https---example.com-user-repo1.git-abc1234 reset --hard abc1234", repositories.GiltDir), - fmt.Sprintf("git -C %s/https---example.com-user-repo.git-abc1234 checkout-index --force --all --prefix %s", + fmt.Sprintf("git -C %s/https---example.com-user-repo1.git-abc1234 checkout-index --force --all --prefix %s", repositories.GiltDir, (dstDir + string(os.PathSeparator))), + fmt.Sprintf("git clone https://example.com/user/repo2.git %s/https---example.com-user-repo2.git-abc1234", + repositories.GiltDir), + fmt.Sprintf("git -C %s/https---example.com-user-repo2.git-abc1234 reset --hard abc1234", + repositories.GiltDir), } assert.Equal(suite.T(), want, got) diff --git a/repositories/repositories_test.go b/repositories/repositories_test.go index 3c580cf..77b4542 100644 --- a/repositories/repositories_test.go +++ b/repositories/repositories_test.go @@ -70,22 +70,132 @@ foo: assert.Equal(suite.T(), want, err) } -func (suite *RepositoriesTestSuite) TestValidateWithoutStringReturnsError() { +func (suite *RepositoriesTestSuite) TestValidateRequiredTopLevelKeysReturnsError() { data := ` --- -- url: - version: - dst: +- missing: + required: + keys: ` jsonData, _ := yaml.YAMLToJSON([]byte(data)) err := suite.r.validate([]byte(jsonData)) - assert.Error(suite.T(), err) + messages := []string{ + "git: git is required", + "version: version is required", + "dstDir: dstDir is required", + } + + for _, want := range messages { + assert.Contains(suite.T(), err.Error(), want) + } +} + +func (suite *RepositoriesTestSuite) TestValidateRequiredSourcesKeysReturnsError() { + data := ` +--- +- git: https://example.com/user/repo.git + version: abc1234 + sources: + - missing: + required: + keys: +` + jsonData, _ := yaml.YAMLToJSON([]byte(data)) + err := suite.r.validate([]byte(jsonData)) messages := []string{ - "0.dst: Invalid type. Expected: string, given: null", - "0.url: Invalid type. Expected: string, given: null", + "src: src is required", + "dstFile: dstFile is required", + } + + for _, want := range messages { + assert.Contains(suite.T(), err.Error(), want) + } +} + +func (suite *RepositoriesTestSuite) TestValidateNoAdditionalTopLevelKeysReturnsError() { + data := ` +--- +- git: https://example.com/user/repo.git + version: abc1234 + dstDir: path/user.repo + extra: +` + jsonData, _ := yaml.YAMLToJSON([]byte(data)) + err := suite.r.validate([]byte(jsonData)) + want := "extra: Additional property extra is not allowed" + + assert.Equal(suite.T(), want, err.Error()) +} + +func (suite *RepositoriesTestSuite) TestValidateNoAdditionalSourcesKeysReturnsError() { + data := ` +--- +- git: https://example.com/user/repo.git + version: abc1234 + sources: + - src: foo + dstFile: bar + extra: +` + jsonData, _ := yaml.YAMLToJSON([]byte(data)) + err := suite.r.validate([]byte(jsonData)) + want := "extra: Additional property extra is not allowed" + + assert.Equal(suite.T(), want, err.Error()) +} + +func (suite *RepositoriesTestSuite) TestValidateMutuallyExclusiveSourcesKeysReturnsError() { + data := ` +--- +- git: https://example.com/user/repo.git + version: abc1234 + sources: + - src: foo + dstFile: bar + dstDir: bar +` + jsonData, _ := yaml.YAMLToJSON([]byte(data)) + err := suite.r.validate([]byte(jsonData)) + want := "0.sources.0: Must validate one and only one schema (oneOf)" + + assert.Equal(suite.T(), want, err.Error()) +} + +func (suite *RepositoriesTestSuite) TestValidateWithoutValueReturnsError() { + data := ` +--- +- git: + version: + dstDir: + +- git: + version: + sources: + - src: + dstFile: + +- git: + version: + sources: + - src: + dstDir: +` + jsonData, _ := yaml.YAMLToJSON([]byte(data)) + err := suite.r.validate([]byte(jsonData)) + messages := []string{ + "0.git: Invalid type. Expected: string, given: null", "0.version: Invalid type. Expected: string, given: null", + "0.dstDir: Invalid type. Expected: string, given: null", + "1.git: Invalid type. Expected: string, given: null", + "1.version: Invalid type. Expected: string, given: null", + "1.sources.0.src: Invalid type. Expected: string, given: null", + "1.sources.0.dstFile: Invalid type. Expected: string, given: null", + "2.git: Invalid type. Expected: string, given: null", + "2.version: Invalid type. Expected: string, given: null", + "2.sources.0.src: Invalid type. Expected: string, given: null", + "2.sources.0.dstDir: Invalid type. Expected: string, given: null", } for _, want := range messages { @@ -93,12 +203,29 @@ func (suite *RepositoriesTestSuite) TestValidateWithoutStringReturnsError() { } } +func (suite *RepositoriesTestSuite) TestValidateMutuallyExclusiveReturnsError() { + data := ` +--- +- git: https://example.com/user/repo.git + version: abc1234 + dstDir: path/user.repo + sources: + - src: foo + dstFile: bar +` + jsonData, _ := yaml.YAMLToJSON([]byte(data)) + err := suite.r.validate([]byte(jsonData)) + want := "0: Must validate one and only one schema (oneOf)" + + assert.Equal(suite.T(), want, err.Error()) +} + func (suite *RepositoriesTestSuite) TestValidate() { data := ` --- -- url: https://example.com/user/repo.git +- git: https://example.com/user/repo.git version: abc1234 - dst: path/user.repo + dstDir: path/user.repo ` jsonData, _ := yaml.YAMLToJSON([]byte(data)) err := suite.r.validate([]byte(jsonData)) diff --git a/repositories/schema.go b/repositories/schema.go index 9ccf734..ad9f5e1 100644 --- a/repositories/schema.go +++ b/repositories/schema.go @@ -27,35 +27,64 @@ const configSchema = ` "minItems": 1, "items": { "type": "object", + "additionalProperties": false, "properties": { - "url": { + "git": { "type": "string" }, "version": { "type": "string", "pattern": "^[0-9a-f]{5,40}$" }, - "dst": { + "dstDir": { "type": "string" }, - "files": { - "type": "array" + "sources": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "type": "string" + }, + "dstFile": { + "type": "string" + }, + "dstDir": { + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "src", + "dstFile" + ] + }, + { + "required": [ + "src", + "dstDir" + ] + } + ] + } } }, - "additionalProperties": true, "oneOf": [ { "required": [ - "url", + "git", "version", - "dst" + "dstDir" ] }, { "required": [ - "url", + "git", "version", - "files" + "sources" ] } ] diff --git a/repository/repository.go b/repository/repository.go index 21beb47..34d6fdf 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -21,24 +21,32 @@ package repository import ( + "errors" "fmt" + "os" "path/filepath" "strings" + + "github.com/logrusorgru/aurora" + "github.com/retr0h/go-gilt/util" ) +// Sources mapping of files and/or directories needing copied. +type Sources struct { + Src string `yaml:"src"` // Src source file or directory to copy. + DstFile string `yaml:"dstFile"` // DstFile destination of file copy. + DstDir string `yaml:"dstDir"` // DstDir destination of directory copy. +} + // Repository containing the repository's details. type Repository struct { - URL string `yaml:"url"` - Version string `yaml:"version"` - Dst string `yaml:"dst"` - GiltDir string // GiltDir option set from CLI. + Git string `yaml:"git"` // Git url of Git repository to clone. + Version string `yaml:"version"` // Version of Git repository to use. + DstDir string `yaml:"dstDir"` // DstDir destination directory to copy clone to. + Sources []Sources // Sources containing files and/or directories to copy. + GiltDir string // GiltDir option set from CLI. } -// // NewRepository factory to create a new Repository instance. -// func NewRepository(debug bool) *Git { -// return &Repository{} -// } - // GetCloneDir returns the path to the Repository's clone directory. func (r *Repository) GetCloneDir() string { return filepath.Join(r.GiltDir, r.getCloneHash()) @@ -49,7 +57,57 @@ func (r *Repository) getCloneHash() string { "/", "-", ":", "-", ) - replacedGitURL := replacer.Replace(r.URL) + replacedGitURL := replacer.Replace(r.Git) return fmt.Sprintf("%s-%s", replacedGitURL, r.Version) } + +// CopySources copy Repository.Src to Repository.DstFile or Repository.DstDir. +func (r *Repository) CopySources() error { + cloneDir := r.GetCloneDir() + + msg := fmt.Sprintf("%-2s [%s]:", "", aurora.Magenta(cloneDir)) + fmt.Println(msg) + + for _, rSource := range r.Sources { + srcFullPath := filepath.Join(cloneDir, rSource.Src) + globbedSrc, err := filepath.Glob(srcFullPath) + if err != nil { + return err + } + + for _, src := range globbedSrc { + // The source is a file. + if info, err := os.Stat(src); err == nil && info.Mode().IsRegular() { + // ... and the destination is declared a directory. + if rSource.DstFile != "" { + if err := util.CopyFile(src, rSource.DstFile); err != nil { + return err + } + } else if rSource.DstDir != "" { + // ... and the destination directory exists. + if info, err := os.Stat(rSource.DstDir); err == nil && info.Mode().IsDir() { + srcBaseFile := filepath.Base(src) + newDst := filepath.Join(rSource.DstDir, srcBaseFile) + if err := util.CopyFile(src, newDst); err != nil { + return err + } + } else { + msg := fmt.Sprintf("DstDir '%s' does not exist", rSource.DstDir) + return errors.New(msg) + } + } + // The source is a directory. + } else if info, err := os.Stat(src); err == nil && info.Mode().IsDir() { + if info, err := os.Stat(rSource.DstDir); err == nil && info.Mode().IsDir() { + os.RemoveAll(rSource.DstDir) + } + if err := util.CopyDir(src, rSource.DstDir); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/repository/repository_public_test.go b/repository/repository_public_test.go index 7741293..42eea99 100644 --- a/repository/repository_public_test.go +++ b/repository/repository_public_test.go @@ -35,9 +35,9 @@ type RepositoryTestSuite struct { func (suite *RepositoryTestSuite) SetupTest() { suite.r = repository.Repository{ - URL: "https://example.com/user/repo.git", + Git: "https://example.com/user/repo.git", Version: "abc1234", - Dst: "path/user.repo", + DstDir: "path/user.repo", GiltDir: "/tmp/gilt", } } diff --git a/test/gilt.yml b/test/gilt.yml index 9b2ea5e..f61b088 100644 --- a/test/gilt.yml +++ b/test/gilt.yml @@ -1,4 +1,16 @@ --- -- url: https://github.com/retr0h/ansible-etcd.git +- git: https://github.com/retr0h/ansible-etcd.git version: 77a95b7 - dst: /tmp/retr0h.ansible-etcd + dstDir: /tmp/retr0h.ansible-etcd + +- git: https://github.com/lorin/openstack-ansible-modules.git + version: 2677cc3 + sources: + - src: "*_manage" + dstDir: library + - src: nova_quota + dstDir: library + - src: neutron_router + dstFile: library/neutron_router.py + - src: tests + dstDir: tests diff --git a/test/integration/test_cli.bats b/test/integration/test_cli.bats index 7630e30..1d53220 100644 --- a/test/integration/test_cli.bats +++ b/test/integration/test_cli.bats @@ -19,12 +19,32 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +GILT_TEST_BASE_DIR=test/integration/tmp +GILT_LIBRARY_DIR=${GILT_TEST_BASE_DIR}/library +GILT_ROLES_DIR=${GILT_TEST_BASE_DIR}/roles +GILT_TEST_DIR=${GILT_TEST_BASE_DIR}/tests +GILT_PROGRAM="../../../main.go" +GILT_DIR=~/.gilt/clone + setup() { - GILT_CLONED_ETCD_REPO=~/.gilt/clone/cache/https---github.com-retr0h-ansible-etcd.git-77a95b7 - GILT_DST_ETCD_REPO=/tmp/retr0h.ansible-etcd + GILT_CLONED_REPO_1=${GILT_DIR}/cache/https---github.com-retr0h-ansible-etcd.git-77a95b7 + GILT_CLONED_REPO_2=${GILT_DIR}/cache/https---github.com-lorin-openstack-ansible-modules.git-2677cc3 + GILT_CLONED_REPO_1_DST_DIR=/tmp/retr0h.ansible-etcd - rm -rf ${GILT_CLONED_ETCD_REPO} - rm -rf ${GILT_DST_ETCD_REPO} + mkdir -p ${GILT_LIBRARY_DIR} + mkdir -p ${GILT_ROLES_DIR} + cp test/gilt.yml ${GILT_TEST_BASE_DIR}/gilt.yml +} + +teardown() { + rm -rf ${GILT_CLONED_REPO_1} + rm -rf ${GILT_CLONED_REPO_2} + rm -rf ${GILT_CLONED_REPO_1_DST_DIR} + + rm -rf ${GILT_LIBRARY_DIR} + rm -rf ${GILT_ROLES_DIR} + rm -rf ${GILT_TEST_DIR} + rm -f ${GILT_TEST_BASE_DIR}/gilt.yml } @test "invoke gilt without arguments prints usage" { @@ -45,35 +65,83 @@ setup() { } @test "invoke gilt overlay subcommand" { - run bash -c 'cd test; go run ../main.go overlay' + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay" [ "$status" -eq 0 ] - - run stat ${GILT_CLONED_ETCD_REPO} - - [ "$status" = 0 ] - - run stat ${GILT_DST_ETCD_REPO} - - [ "$status" = 0 ] } @test "invoke gilt overlay subcommand with filename flag" { - run go run main.go overlay --filename test/gilt.yml + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay --filename gilt.yml" [ "$status" -eq 0 ] } @test "invoke gilt overlay subcommand with f flag" { - run go run main.go overlay -f test/gilt.yml + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay -f gilt.yml" [ "$status" -eq 0 ] } @test "invoke gilt overlay subcommand with debug flag" { - run go run main.go --debug overlay --filename test/gilt.yml + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} --debug overlay" [ "$status" -eq 0 ] echo "${output}" | grep "[https://github.com/retr0h/ansible-etcd.git@77a95b7]" echo "${output}" | grep -E ".*Cloning to.*https---github.com-retr0h-ansible-etcd.git-77a95b7" } + +@test "invoke gilt overlay when already cloned" { + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay" + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay" + + echo "${output}" | grep "Clone already exists" +} + +@test "invoke gilt overlay and clone" { + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay" + + run stat ${GILT_CLONED_REPO_1} + [ "$status" = 0 ] + + run stat ${GILT_CLONED_REPO_2} + [ "$status" = 0 ] +} + +@test "invoke gilt overlay and checkout index" { + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay" + + run stat ${GILT_CLONED_REPO_1_DST_DIR} + [ "$status" = 0 ] +} + +@test "invoke gilt overlay and copy sources" { + run bash -c "cd ${GILT_TEST_BASE_DIR}; go run ${GILT_PROGRAM} --giltdir ${GILT_DIR} overlay" + + # Copy src file matched by regexp to dst dir. + run stat ${GILT_LIBRARY_DIR}/cinder_manage + [ "$status" = 0 ] + run stat ${GILT_LIBRARY_DIR}/glance_manage + [ "$status" = 0 ] + run stat ${GILT_LIBRARY_DIR}/heat_manage + [ "$status" = 0 ] + run stat ${GILT_LIBRARY_DIR}/keystone_manage + [ "$status" = 0 ] + run stat ${GILT_LIBRARY_DIR}/nova_manage + [ "$status" = 0 ] + + # Copy src file to dst dir. + run stat ${GILT_LIBRARY_DIR}/nova_quota + [ "$status" = 0 ] + + # Copy src file to dst file. + run stat ${GILT_LIBRARY_DIR}/neutron_router.py + [ "$status" = 0 ] + + # Copy src dir to dst dir. + run stat ${GILT_TEST_DIR}/keystone_service.py + echo $output + [ "$status" = 0 ] + run stat ${GILT_TEST_DIR}/test_keystone_service.py + echo $output + [ "$status" = 0 ] +} diff --git a/test/resources/copy/dir/foo b/test/resources/copy/dir/foo new file mode 100644 index 0000000..e69de29 diff --git a/test/resources/copy/file b/test/resources/copy/file new file mode 100644 index 0000000..e69de29 diff --git a/util/util.go b/util/util.go index 7b58f9b..503c036 100644 --- a/util/util.go +++ b/util/util.go @@ -31,6 +31,7 @@ import ( "strings" "github.com/logrusorgru/aurora" + "github.com/retr0h/go-gilt/copy" ) var ( @@ -94,3 +95,30 @@ func RunCmd(debug bool, name string, args ...string) error { return nil } + +// CopyFile copies src file to dst. +func CopyFile(src string, dst string) error { + baseSrc := filepath.Base(src) + msg := fmt.Sprintf("%-4s - Copying file '%s' to '%s'", "", aurora.Cyan(baseSrc), aurora.Cyan(dst)) + fmt.Println(msg) + + if err := copy.File(src, dst); err != nil { + return err + } + + return nil +} + +// CopyDir copies src directory to dst. +func CopyDir(src string, dst string) error { + baseSrc := filepath.Base(src) + msg := fmt.Sprintf("%-4s - Copying dir '%s' to '%s'", "", aurora.Cyan(baseSrc), aurora.Cyan(dst)) + fmt.Println(msg) + + if err := copy.Dir(src, dst); err != nil { + fmt.Println(err) + return err + } + + return nil +} diff --git a/util/util_public_test.go b/util/util_public_test.go index c3bb753..01c3ea5 100644 --- a/util/util_public_test.go +++ b/util/util_public_test.go @@ -21,7 +21,9 @@ package util_test import ( + "os" "os/user" + "path" "path/filepath" "testing" @@ -89,3 +91,39 @@ func TestRunCommandPrintsStreamingStderr(t *testing.T) { assert.Equal(t, want, got) } + +func TestCopyFile(t *testing.T) { + srcFile := path.Join("..", "test", "resources", "copy", "file") + dstFile := path.Join("..", "test", "resources", "copy", "copiedFile") + err := util.CopyFile(srcFile, dstFile) + defer os.Remove(dstFile) + + assert.NoError(t, err) + assert.FileExistsf(t, dstFile, "File does not exist") +} + +func TestCopyFileReturnsError(t *testing.T) { + srcFile := path.Join("..", "test", "resources", "copy", "file") + invalidDstFile := "/super/invalid/path/to/write/to" + err := util.CopyFile(srcFile, invalidDstFile) + + assert.Error(t, err) +} + +func TestCopyDir(t *testing.T) { + srcDir := path.Join("..", "test", "resources", "copy", "dir") + dstDir := path.Join("..", "test", "resources", "copy", "copiedDir") + err := util.CopyDir(srcDir, dstDir) + defer os.RemoveAll(dstDir) + + assert.NoError(t, err) + assert.DirExistsf(t, dstDir, "Dir does not exist") +} + +func TestCopyDirReturnsError(t *testing.T) { + srcDir := path.Join("..", "test", "resources", "copy", "dir") + invalidDstDir := "/super/invalid/path/to/write/to" + err := util.CopyDir(srcDir, invalidDstDir) + + assert.Error(t, err) +}