diff --git a/.gitignore b/.gitignore index a4bb3e7acb97..fa7f133d599f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist/ # delve debug binaries cmd/**/debug debug.test +*.iml diff --git a/api/workflow/v1alpha1/types.go b/api/workflow/v1alpha1/types.go index 36cfd886d89d..49114e85f429 100644 --- a/api/workflow/v1alpha1/types.go +++ b/api/workflow/v1alpha1/types.go @@ -171,9 +171,10 @@ type Artifact struct { // It is also used to describe the location of multiple artifacts such as the archive location // of a single workflow step, which the executor will use as a default location to store its files. type ArtifactLocation struct { - S3 *S3Artifact `json:"s3,omitempty"` - Git *GitArtifact `json:"git,omitempty"` - HTTP *HTTPArtifact `json:"http,omitempty"` + S3 *S3Artifact `json:"s3,omitempty"` + Git *GitArtifact `json:"git,omitempty"` + HTTP *HTTPArtifact `json:"http,omitempty"` + Artifactory *ArtifactoryArtifact `json:"artifactory,omitempty"` } type Outputs struct { @@ -330,6 +331,16 @@ type GitArtifact struct { PasswordSecret *apiv1.SecretKeySelector `json:"passwordSecret,omitempty"` } +type ArtifactoryAuth struct { + UsernameSecret *apiv1.SecretKeySelector `json:"usernameSecret,omitempty"` + PasswordSecret *apiv1.SecretKeySelector `json:"passwordSecret,omitempty"` +} + +type ArtifactoryArtifact struct { + ArtifactoryAuth `json:",inline,squash"` + URL string `json:"url"` +} + type HTTPArtifact struct { URL string `json:"url"` } @@ -426,5 +437,5 @@ func (args *Arguments) GetParameterByName(name string) *Parameter { // HasLocation whether or not an artifact has a location defined func (a *Artifact) HasLocation() bool { - return a.S3 != nil || a.Git != nil || a.HTTP != nil + return a.S3 != nil || a.Git != nil || a.HTTP != nil || a.Artifactory != nil } diff --git a/examples/artifactory-artifact.yaml b/examples/artifactory-artifact.yaml new file mode 100644 index 000000000000..6f6fd549098c --- /dev/null +++ b/examples/artifactory-artifact.yaml @@ -0,0 +1,59 @@ +# This example demonstrates the use of artifactory as the store for artifacts. This example assumes the following: +# 1. you have artifactory running in the same namespace as where this workflow will be run and you have created a repo with the name "generic-local" +# 2. you have created a kubernetes secret for storing artifactory username/password. To create kubernetes secret required for this example, +# run the following command: +# $ kubectl create secret generic my-artifactory-credentials --from-literal=username= --from-literal=password= + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: artifactory-artifact- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + - - name: consume-artifact + template: print-message + arguments: + artifacts: + - name: message + from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}" + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world.txt"] + outputs: + artifacts: + - name: hello-art + path: /tmp/hello_world.txt + artifactory: + url: http://artifactory:8081/artifactory/generic-local/hello_world.tgz + usernameSecret: + name: my-artifactory-credentials + key: username + passwordSecret: + name: my-artifactory-credentials + key: password + + - name: print-message + inputs: + artifacts: + - name: message + path: /tmp/message + artifactory: + url: http://artifactory:8081/artifactory/generic-local/hello_world.tgz + usernameSecret: + name: my-artifactory-credentials + key: username + passwordSecret: + name: my-artifactory-credentials + key: password + container: + image: alpine:latest + command: [sh, -c] + args: ["cat /tmp/message"] diff --git a/workflow/artifacts/artifactory/artifactory.go b/workflow/artifacts/artifactory/artifactory.go new file mode 100644 index 000000000000..4f46a2468eac --- /dev/null +++ b/workflow/artifacts/artifactory/artifactory.go @@ -0,0 +1,66 @@ +package artifactory + +import ( + wfv1 "github.com/argoproj/argo/api/workflow/v1alpha1" + "net/http" + "os" + "github.com/argoproj/argo/errors" + "io" +) + +type ArtifactoryArtifactDriver struct { + Username string + Password string +} + + +// Download artifact from an artifactory URL +func (a *ArtifactoryArtifactDriver) Load(artifact *wfv1.Artifact, path string) error { + + lf, err := os.Create(path) + if err != nil { + return err + } + defer lf.Close() + + req,err := http.NewRequest(http.MethodGet,artifact.Artifactory.URL,nil) + if err != nil { + return err + } + req.SetBasicAuth(a.Username,a.Password) + res, err := (&http.Client{}).Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return errors.InternalErrorf("loading file from artifactory failed with reason:%s",res.Status) + } + + _,err = io.Copy(lf,res.Body) + + return err +} + +// UpLoad artifact to an artifactory URL +func (a *ArtifactoryArtifactDriver) Save(path string, artifact *wfv1.Artifact) error { + + f, err := os.Open(path) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPut,artifact.Artifactory.URL,f) + if err != nil { + return err + } + req.SetBasicAuth(a.Username,a.Password) + res, err := (&http.Client{}).Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return errors.InternalErrorf("saving file %s to artifactory failed with reason:%s",path,res.Status) + } + return nil +} diff --git a/workflow/artifacts/artifactory/artifactory_test.go b/workflow/artifacts/artifactory/artifactory_test.go new file mode 100644 index 000000000000..4d180a6a4b80 --- /dev/null +++ b/workflow/artifacts/artifactory/artifactory_test.go @@ -0,0 +1,57 @@ +package artifactory_test + +import ( + art "../artifactory" + wfv1 "github.com/argoproj/argo/api/workflow/v1alpha1" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "testing" + "time" +) + +const ( + LoadFileName string = "argo_artifactory_test_load.txt" + SaveFileName string = "argo_artifactory_test_save.txt" + RepoName string = "generic-local" + URL string = "http://localhost:8081/artifactory/" + RepoName + "/" + LoadFileName + Username string = "admin" + Password string = "password" +) + +func TestSaveAndLoad(t *testing.T) { + + t.Skip("This test is skipped since it depends on external service") + fileContent := "time: " + string(time.Now().UnixNano()) + + // create file to test save + lf, err := ioutil.TempFile("", LoadFileName) + assert.Nil(t, err) + defer os.Remove(lf.Name()) + // load file with test content + content := []byte(fileContent) + _, err = lf.Write(content) + assert.Nil(t, err) + err = lf.Close() + assert.Nil(t, err) + + // create file to test load + sf, err := ioutil.TempFile("", SaveFileName) + assert.Nil(t, err) + defer os.Remove(sf.Name()) + + artL := &wfv1.Artifact{} + artL.Artifactory = &wfv1.ArtifactoryArtifact{ + URL: URL, + } + driver := &art.ArtifactoryArtifactDriver{ + Username: Username, + Password: Password, + } + driver.Save(lf.Name(), artL) + driver.Load(artL, sf.Name()) + + dat, err := ioutil.ReadFile(sf.Name()) + assert.Nil(t, err) + assert.Equal(t, fileContent, string(dat)) +} diff --git a/workflow/controller/controller.go b/workflow/controller/controller.go index 19fbf2f55f47..e0573e802064 100644 --- a/workflow/controller/controller.go +++ b/workflow/controller/controller.go @@ -79,6 +79,7 @@ const ( type ArtifactRepository struct { S3 *S3ArtifactRepository `json:"s3,omitempty"` // Future artifact repository support here + Artifactory *ArtifactoryArtifactRepository `json:"artifactory,omitempty"` } type S3ArtifactRepository struct { wfv1.S3Bucket `json:",inline,squash"` @@ -87,6 +88,12 @@ type S3ArtifactRepository struct { KeyPrefix string `json:"keyPrefix,omitempty"` } +type ArtifactoryArtifactRepository struct { + wfv1.ArtifactoryAuth `json:",inline,squash"` + // RepoUrl is the url for artifactory repo . + RepoUrl string `json:"RepoUrl,omitempty"` +} + // NewWorkflowController instantiates a new WorkflowController func NewWorkflowController(config *rest.Config, configMap string) *WorkflowController { // make a new config for our extension's API group, using the first config as a baseline diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index 82a2c9389f5b..15e6f3fc6396 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -424,9 +424,21 @@ func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Templat S3Bucket: woc.controller.Config.ArtifactRepository.S3.S3Bucket, Key: artLocationKey, } + } else if woc.controller.Config.ArtifactRepository.Artifactory != nil { + log.Debugf("Setting artifactory artifact repository information") + repoUrl := "" + if woc.controller.Config.ArtifactRepository.Artifactory.RepoUrl != "" { + repoUrl = woc.controller.Config.ArtifactRepository.Artifactory.RepoUrl + "/" + } + artUrl := fmt.Sprintf("%s%s/%s", repoUrl, woc.wf.ObjectMeta.Name, pod.ObjectMeta.Name) + tmpl.ArchiveLocation.Artifactory = &wfv1.ArtifactoryArtifact{ + ArtifactoryAuth: woc.controller.Config.ArtifactRepository.Artifactory.ArtifactoryAuth, + URL: artUrl, + } } else { for _, art := range tmpl.Outputs.Artifacts { if !art.HasLocation() { + log.Errorf("artifact has no location details:%#v", art) return errors.Errorf(errors.CodeBadRequest, "controller is not configured with a default archive location") } } diff --git a/workflow/executor/executor.go b/workflow/executor/executor.go index a00bf49e3350..b8670c6b8584 100644 --- a/workflow/executor/executor.go +++ b/workflow/executor/executor.go @@ -15,6 +15,7 @@ import ( wfv1 "github.com/argoproj/argo/api/workflow/v1alpha1" "github.com/argoproj/argo/errors" artifact "github.com/argoproj/argo/workflow/artifacts" + "github.com/argoproj/argo/workflow/artifacts/artifactory" "github.com/argoproj/argo/workflow/artifacts/git" "github.com/argoproj/argo/workflow/artifacts/http" "github.com/argoproj/argo/workflow/artifacts/s3" @@ -169,6 +170,10 @@ func (we *WorkflowExecutor) SaveArtifacts() error { shallowCopy := *we.Template.ArchiveLocation.S3 art.S3 = &shallowCopy art.S3.Key = path.Join(art.S3.Key, fileName) + } else if we.Template.ArchiveLocation.Artifactory != nil { + shallowCopy := *we.Template.ArchiveLocation.Artifactory + art.Artifactory = &shallowCopy + art.Artifactory.URL = path.Join(art.Artifactory.URL, fileName) } else { return errors.Errorf(errors.CodeBadRequest, "Unable to determine path to store %s. Archive location provided no information", art.Name) } @@ -280,6 +285,24 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv return &gitDriver, nil } + if art.Artifactory != nil { + // Getting Kubernetes namespace from the environment variables + namespace := os.Getenv(common.EnvVarNamespace) + username, err := we.getSecrets(namespace, art.Artifactory.UsernameSecret.Name, art.Artifactory.UsernameSecret.Key) + if err != nil { + return nil, err + } + password, err := we.getSecrets(namespace, art.Artifactory.PasswordSecret.Name, art.Artifactory.PasswordSecret.Key) + if err != nil { + return nil, err + } + driver := artifactory.ArtifactoryArtifactDriver{ + Username: username, + Password: password, + } + return &driver, nil + + } return nil, errors.Errorf(errors.CodeBadRequest, "Unsupported artifact driver for %s", art.Name) }