Skip to content

Commit

Permalink
Ability to copy files out of the source repo
Browse files Browse the repository at this point in the history
With the following gilt.yml, Gilt can copy the src to the dst.

    - git: https://github.com/lorin/openstack-ansible-modules.git
      version: 2677cc3
      sources:
	- src: "*_manage"
	  dstDir: library
	- src: nova_quota
	  dstDir: library
	- src: neutron_router
	  dstFile: library/neutron_router.py
	- src: tests
	  dstDir: tests

Reworked some of the previous schema as well to be more consistent
existing naming.
  • Loading branch information
retr0h committed Oct 5, 2018
1 parent a51f1ed commit e187d54
Show file tree
Hide file tree
Showing 21 changed files with 704 additions and 134 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
.build/
coverage*
test/integration/tmp/
19 changes: 12 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
VENDOR := vendor
PKGS := $(shell go list ./... | grep -v /$(VENDOR)/)
SRC = $(shell find . -type f -name '*.go' -not -path "./$(VENDOR)/*")
$(if $(PKGS), , $(error "go list failed"))
PKGS_DELIM := $(shell echo $(PKGS) | sed -e 's/ /,/g')
GITCOMMIT ?= $(shell git rev-parse --short HEAD)
Expand All @@ -19,9 +20,9 @@ LDFLAGS := -s \
BUILDDIR := .build
export GOCACHE = off

test: fmt lint vet bats
test: fmtcheck lint vet bats
@echo "+ $@"
@go test -parallel 5 -covermode=count ./...
@go test -tags=integration -parallel 5 -covermode=count ./...

bats:
@echo "+ $@"
Expand All @@ -31,22 +32,26 @@ cover:
@echo "+ $@"
$(shell [ -e coverage.out ] && rm coverage.out)
@echo "mode: count" > coverage-all.out
$(foreach pkg,$(PKGS),\
go test -coverprofile=coverage.out -covermode=count $(pkg);\
@$(foreach pkg,$(PKGS),\
go test -tags=integration -coverprofile=coverage.out -covermode=count $(pkg);\
tail -n +2 coverage.out >> coverage-all.out;)
@go tool cover -html=coverage-all.out -o=coverage-all.html

fmt:
@echo "+ $@"
@gofmt -s -l . | grep -v $(VENDOR) | tee /dev/stderr
@gofmt -s -l -w $(SRC)

fmtcheck:
@echo "+ $@"
@bash -c "diff -u <(echo -n) <(gofmt -d $(SRC))"

lint:
@echo "+ $@"
@echo $(PKGS) | xargs -L1 golint -set_exit_status | tee /dev/stderr
@echo $(PKGS) | xargs -L1 golint -set_exit_status

vet:
@echo "+ $@"
@go vet $(shell go list ./... | grep -v $(VENDOR))
@go vet $(PKGS)

clean:
@echo "+ $@"
Expand Down
20 changes: 16 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,26 @@ not 100% compatible with the python version, and not yet complete.

Create the giltfile (`gilt.yml`).

Clone the specified `url`@`version` to the configurable path `--giltdir`, and
extract the repository to the provided `dst`.
Clone the specified `url`@`version` to the configurable path `--giltdir`.
Extract the repo the `dstDir` when `dstDir` is provided. Otherwise, copy files
and/or directories to the desired destinations.

```yaml
---
- url: https://github.com/retr0h/ansible-etcd.git
- git: https://github.com/retr0h/ansible-etcd.git
version: 77a95b7
dst: roles/retr0h.ansible-etcd
dstDir: roles/retr0h.ansible-etcd
- git: https://github.com/lorin/openstack-ansible-modules.git
version: 2677cc3
sources:
- src: "*_manage"
dstDir: library
- src: nova_quota
dstDir: library
- src: neutron_router
dstFile: library/neutron_router.py
- src: tests
dstDir: tests
```

Overlay a remote repository into the destination provided.
Expand Down
145 changes: 145 additions & 0 deletions copy/copy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
// Copyright (c) 2018 John Dewey

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

// Package copy taken from terraform, and added as copy package to avoid testing
// directly and dropping coverage of util package.
// https://github.com/hashicorp/terraform/blob/master/helper/copy/copy.go
package copy

import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
)

// From: https://gist.github.com/m4ng0squ4sh/92462b38df26839a3ca324697c8cba04

// File copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file. The file mode will be copied from the source and
// the copied data is synced/flushed to stable storage.
func File(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()

out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
if e := out.Close(); e != nil {
err = e
}
}()

_, err = io.Copy(out, in)
if err != nil {
return
}

err = out.Sync()
if err != nil {
return
}

si, err := os.Stat(src)
if err != nil {
return
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return
}

return
}

// Dir recursively copies a directory tree, attempting to preserve permissions.
// Source directory must exist, destination directory must *not* exist.
// Symlinks are ignored and skipped.
func Dir(src string, dst string) (err error) {
src = filepath.Clean(src)
dst = filepath.Clean(dst)

si, err := os.Stat(src)
if err != nil {
return err
}
if !si.IsDir() {
return fmt.Errorf("source is not a directory")
}

_, err = os.Stat(dst)
if err != nil && !os.IsNotExist(err) {
return
}
if err == nil {
return fmt.Errorf("destination already exists")
}

err = os.MkdirAll(dst, si.Mode())
if err != nil {
return
}

entries, err := ioutil.ReadDir(src)
if err != nil {
return
}

for _, entry := range entries {
srcPath := filepath.Join(src, entry.Name())
dstPath := filepath.Join(dst, entry.Name())

// If the entry is a symlink, we copy the contents
for entry.Mode()&os.ModeSymlink != 0 {

target, err := filepath.EvalSymlinks(srcPath)
if err != nil {
return err
}

entry, err = os.Stat(target)
if err != nil {
return err
}
}

if entry.IsDir() {
err = Dir(srcPath, dstPath)
if err != nil {
return
}
} else {
err = File(srcPath, dstPath)
if err != nil {
return
}
}
}

return
}
17 changes: 9 additions & 8 deletions git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,15 @@ func NewGit(debug bool) *Git {
}
}

// Clone clone Repository.URL to Repository.getCloneDir, and hard checkout
// Clone clone Repository.Git to Repository.getCloneDir, and hard checkout
// to Repository.Version.
func (g *Git) Clone(repository repository.Repository) error {
cloneDir := repository.GetCloneDir()

msg := fmt.Sprintf("[%s@%s]:", aurora.Magenta(repository.URL), aurora.Magenta(repository.Version))
msg := fmt.Sprintf("[%s@%s]:", aurora.Magenta(repository.Git), aurora.Magenta(repository.Version))
fmt.Println(msg)

msg = fmt.Sprintf("%-2s - %s '%s'", "", aurora.Cyan("Cloning to"), aurora.Cyan(cloneDir))
msg = fmt.Sprintf("%-2s - Cloning to '%s'", "", aurora.Cyan(cloneDir))
fmt.Println(msg)

if _, err := os.Stat(cloneDir); os.IsNotExist(err) {
Expand All @@ -67,7 +67,8 @@ func (g *Git) Clone(repository repository.Repository) error {
return err
}
} else {
msg := fmt.Sprintf("%-4s * %s", "", aurora.Brown("Clone already exists"))
bang := aurora.Bold(aurora.Red("!"))
msg := fmt.Sprintf("%-2s %s %s", "", bang, aurora.Brown("Clone already exists"))
fmt.Println(msg)
}

Expand All @@ -76,7 +77,7 @@ func (g *Git) Clone(repository repository.Repository) error {

func (g *Git) clone(repository repository.Repository) error {
cloneDir := repository.GetCloneDir()
err := RunCommand(g.Debug, "git", "clone", repository.URL, cloneDir)
err := RunCommand(g.Debug, "git", "clone", repository.Git, cloneDir)

return err
}
Expand All @@ -88,15 +89,15 @@ func (g *Git) reset(repository repository.Repository) error {
return err
}

// CheckoutIndex checkout Repository.Git to Repository.Dst.
// CheckoutIndex checkout Repository.Git to Repository.DstDir.
func (g *Git) CheckoutIndex(repository repository.Repository) error {
cloneDir := repository.GetCloneDir()
dstDir, err := FilePathAbs(repository.Dst)
dstDir, err := FilePathAbs(repository.DstDir)
if err != nil {
return err
}

msg := fmt.Sprintf("%-2s - %s '%s'", "", aurora.Cyan("Extracting to"), aurora.Cyan(dstDir))
msg := fmt.Sprintf("%-2s - Extracting to '%s'", "", aurora.Cyan(dstDir))
fmt.Println(msg)

cmdArgs := []string{
Expand Down
6 changes: 3 additions & 3 deletions git/git_public_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ type GitTestSuite struct {
func (suite *GitTestSuite) SetupTest() {
suite.g = git.NewGit(false)
suite.r = repository.Repository{
URL: "https://example.com/user/repo.git",
Git: "https://example.com/user/repo.git",
Version: "abc1234",
Dst: "path/user.repo",
DstDir: "path/user.repo",
GiltDir: testutil.CreateTempDirectory(),
}
}
Expand Down Expand Up @@ -136,7 +136,7 @@ func (suite *GitTestSuite) TestCheckoutIndex() {
return err
}

dstDir, _ := git.FilePathAbs(suite.r.Dst)
dstDir, _ := git.FilePathAbs(suite.r.DstDir)
got := git.MockRunCommand(anon)
want := []string{
fmt.Sprintf("git -C %s/https---example.com-user-repo.git-abc1234 checkout-index --force --all --prefix %s",
Expand Down
4 changes: 2 additions & 2 deletions git/git_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ type GitTestSuite struct {
func (suite *GitTestSuite) SetupTest() {
suite.g = NewGit(false)
suite.r = repository.Repository{
URL: "https://example.com/user/repo.git",
Git: "https://example.com/user/repo.git",
Version: "abc1234",
Dst: "path/user.repo",
DstDir: "path/user.repo",
GiltDir: testutil.CreateTempDirectory(),
}
}
Expand Down
1 change: 0 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

// Package main TODO
package main

import (
Expand Down
17 changes: 15 additions & 2 deletions repositories/repositories.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"errors"
"fmt"
"io/ioutil"
"os"
"strings"

"github.com/ghodss/yaml"
Expand Down Expand Up @@ -117,12 +118,24 @@ func (r *Repositories) Overlay() error {
return err
}

// Checkout into repository.Dst.
if repository.Dst != "" {
// Checkout into repository.DstDir.
if repository.DstDir != "" {
// Delete dstDir since Checkout-Index does not clean old files that may
// no longer exist in repository.
if info, err := os.Stat(repository.DstDir); err == nil && info.Mode().IsDir() {
os.RemoveAll(repository.DstDir)
}
if err := g.CheckoutIndex(repository); err != nil {
return err
}
}

// Copy sources from Repository.Src to Repository.DstDir or Repository.DstFile.
if len(repository.Sources) > 0 {
if err := repository.CopySources(); err != nil {
return err
}
}
}

return nil
Expand Down
Loading

0 comments on commit e187d54

Please sign in to comment.