Added support for artifactory

Resolves argoproj#453

.gitignore

@@ -6,3 +6,4 @@ dist/
 # delve debug binaries
 cmd/**/debug
 debug.test
+*.iml

api/workflow/v1alpha1/types.go

@@ -171,9 +171,10 @@ type Artifact struct {
 // It is also used to describe the location of multiple artifacts such as the archive location
 // of a single workflow step, which the executor will use as a default location to store its files.
 type ArtifactLocation struct {
- S3 *S3Artifact `json:"s3,omitempty"`
- Git *GitArtifact `json:"git,omitempty"`
- HTTP *HTTPArtifact `json:"http,omitempty"`
+ S3 *S3Artifact `json:"s3,omitempty"`
+ Git *GitArtifact `json:"git,omitempty"`
+ HTTP *HTTPArtifact `json:"http,omitempty"`
+ Artifactory *ArtifactoryArtifact `json:"artifactory,omitempty"`
 }
 
 type Outputs struct {
@@ -330,6 +331,16 @@ type GitArtifact struct {
  PasswordSecret *apiv1.SecretKeySelector `json:"passwordSecret,omitempty"`
 }
 
+type ArtifactoryAuth struct {
+ UsernameSecret *apiv1.SecretKeySelector `json:"usernameSecret,omitempty"`
+ PasswordSecret *apiv1.SecretKeySelector `json:"passwordSecret,omitempty"`
+}
+
+type ArtifactoryArtifact struct {
+ ArtifactoryAuth `json:",inline,squash"`
+ URL string `json:"url"`
+}
+
 type HTTPArtifact struct {
  URL string `json:"url"`
 }
@@ -426,5 +437,5 @@ func (args *Arguments) GetParameterByName(name string) *Parameter {
 
 // HasLocation whether or not an artifact has a location defined
 func (a *Artifact) HasLocation() bool {
- return a.S3 != nil || a.Git != nil || a.HTTP != nil
+ return a.S3 != nil || a.Git != nil || a.HTTP != nil || a.Artifactory != nil
 }

examples/artifactory-artifact.yaml

@@ -0,0 +1,59 @@
+# This example demonstrates the use of artifactory as the store for artifacts. This example assumes the following:
+# 1. you have artifactory running in the same namespace as where this workflow will be run and you have created a repo with the name "generic-local"
+# 2. you have created a kubernetes secret for storing artifactory username/password. To create kubernetes secret required for this example,
+# run the following command:
+# $ kubectl create secret generic my-artifactory-credentials --from-literal=username=<YOUR-ARTIFACTORY-USERNAME> --from-literal=password=<YOUR-ARTIFACTORY-PASSWORD>
+
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: artifactory-artifact-
+spec:
+ entrypoint: artifact-example
+ templates:
+ - name: artifact-example
+ steps:
+ - - name: generate-artifact
+ template: whalesay
+ - - name: consume-artifact
+ template: print-message
+ arguments:
+ artifacts:
+ - name: message
+ from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}"
+
+ - name: whalesay
+ container:
+ image: docker/whalesay:latest
+ command: [sh, -c]
+ args: ["cowsay hello world | tee /tmp/hello_world.txt"]
+ outputs:
+ artifacts:
+ - name: hello-art
+ path: /tmp/hello_world.txt
+ artifactory:
+ url: http:https://artifactory:8081/artifactory/generic-local/hello_world.tgz
+ usernameSecret:
+ name: my-artifactory-credentials
+ key: username
+ passwordSecret:
+ name: my-artifactory-credentials
+ key: password
+
+ - name: print-message
+ inputs:
+ artifacts:
+ - name: message
+ path: /tmp/message
+ artifactory:
+ url: http:https://artifactory:8081/artifactory/generic-local/hello_world.tgz
+ usernameSecret:
+ name: my-artifactory-credentials
+ key: username
+ passwordSecret:
+ name: my-artifactory-credentials
+ key: password
+ container:
+ image: alpine:latest
+ command: [sh, -c]
+ args: ["cat /tmp/message"]

workflow/artifacts/artifactory/artifactory.go

@@ -0,0 +1,66 @@
+package artifactory
+
+import (
+ wfv1 "github.com/argoproj/argo/api/workflow/v1alpha1"
+ "net/http"
+ "os"
+ "github.com/argoproj/argo/errors"
+ "io"
+)
+
+type ArtifactoryArtifactDriver struct {
+ Username string
+ Password string
+}
+
+
+// Download artifact from an artifactory URL
+func (a *ArtifactoryArtifactDriver) Load(artifact *wfv1.Artifact, path string) error {
+
+ lf, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+ defer lf.Close()
+
+ req,err := http.NewRequest(http.MethodGet,artifact.Artifactory.URL,nil)
+ if err != nil {
+ return err
+ }
+ req.SetBasicAuth(a.Username,a.Password)
+ res, err := (&http.Client{}).Do(req)
+ if err != nil {
+ return err
+ }
+ defer res.Body.Close()
+ if res.StatusCode < 200 || res.StatusCode >= 300 {
+ return errors.InternalErrorf("loading file from artifactory failed with reason:%s",res.Status)
+ }
+
+ _,err = io.Copy(lf,res.Body)
+
+ return err
+}
+
+// UpLoad artifact to an artifactory URL
+func (a *ArtifactoryArtifactDriver) Save(path string, artifact *wfv1.Artifact) error {
+
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ req, err := http.NewRequest(http.MethodPut,artifact.Artifactory.URL,f)
+ if err != nil {
+ return err
+ }
+ req.SetBasicAuth(a.Username,a.Password)
+ res, err := (&http.Client{}).Do(req)
+ if err != nil {
+ return err
+ }
+ defer res.Body.Close()
+ if res.StatusCode < 200 || res.StatusCode >= 300 {
+ return errors.InternalErrorf("saving file %s to artifactory failed with reason:%s",path,res.Status)
+ }
+ return nil
+}

workflow/artifacts/artifactory/artifactory_test.go

@@ -0,0 +1,57 @@
+package artifactory_test
+
+import (
+ art "../artifactory"
+ wfv1 "github.com/argoproj/argo/api/workflow/v1alpha1"
+ "github.com/stretchr/testify/assert"
+ "io/ioutil"
+ "os"
+ "testing"
+ "time"
+)
+
+const (
+ LoadFileName string = "argo_artifactory_test_load.txt"
+ SaveFileName string = "argo_artifactory_test_save.txt"
+ RepoName string = "generic-local"
+ URL string = "http:https://localhost:8081/artifactory/" + RepoName + "/" + LoadFileName
+ Username string = "admin"
+ Password string = "password"
+)
+
+func TestSaveAndLoad(t *testing.T) {
+
+ t.Skip("This test is skipped since it depends on external service")
+ fileContent := "time: " + string(time.Now().UnixNano())
+
+ // create file to test save
+ lf, err := ioutil.TempFile("", LoadFileName)
+ assert.Nil(t, err)
+ defer os.Remove(lf.Name())
+ // load file with test content
+ content := []byte(fileContent)
+ _, err = lf.Write(content)
+ assert.Nil(t, err)
+ err = lf.Close()
+ assert.Nil(t, err)
+
+ // create file to test load
+ sf, err := ioutil.TempFile("", SaveFileName)
+ assert.Nil(t, err)
+ defer os.Remove(sf.Name())
+
+ artL := &wfv1.Artifact{}
+ artL.Artifactory = &wfv1.ArtifactoryArtifact{
+ URL: URL,
+ }
+ driver := &art.ArtifactoryArtifactDriver{
+ Username: Username,
+ Password: Password,
+ }
+ driver.Save(lf.Name(), artL)
+ driver.Load(artL, sf.Name())
+
+ dat, err := ioutil.ReadFile(sf.Name())
+ assert.Nil(t, err)
+ assert.Equal(t, fileContent, string(dat))
+}

workflow/controller/controller.go

@@ -79,6 +79,7 @@ const (
 type ArtifactRepository struct {
  S3 *S3ArtifactRepository `json:"s3,omitempty"`
  // Future artifact repository support here
+ Artifactory *ArtifactoryArtifactRepository `json:"artifactory,omitempty"`
 }
 type S3ArtifactRepository struct {
  wfv1.S3Bucket `json:",inline,squash"`
@@ -87,6 +88,12 @@ type S3ArtifactRepository struct {
  KeyPrefix string `json:"keyPrefix,omitempty"`
 }
 
+type ArtifactoryArtifactRepository struct {
+ wfv1.ArtifactoryAuth `json:",inline,squash"`
+ // RepoUrl is the url for artifactory repo .
+ RepoUrl string `json:"RepoUrl,omitempty"`
+}
+
 // NewWorkflowController instantiates a new WorkflowController
 func NewWorkflowController(config *rest.Config, configMap string) *WorkflowController {
  // make a new config for our extension's API group, using the first config as a baseline

workflow/controller/workflowpod.go

@@ -424,9 +424,21 @@ func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Templat
  S3Bucket: woc.controller.Config.ArtifactRepository.S3.S3Bucket,
  Key: artLocationKey,
  }
+ } else if woc.controller.Config.ArtifactRepository.Artifactory != nil {
+ log.Debugf("Setting artifactory artifact repository information")
+ repoUrl := ""
+ if woc.controller.Config.ArtifactRepository.Artifactory.RepoUrl != "" {
+ repoUrl = woc.controller.Config.ArtifactRepository.Artifactory.RepoUrl + "/"
+ }
+ artUrl := fmt.Sprintf("%s%s/%s", repoUrl, woc.wf.ObjectMeta.Name, pod.ObjectMeta.Name)
+ tmpl.ArchiveLocation.Artifactory = &wfv1.ArtifactoryArtifact{
+ ArtifactoryAuth: woc.controller.Config.ArtifactRepository.Artifactory.ArtifactoryAuth,
+ URL: artUrl,
+ }
  } else {
  for _, art := range tmpl.Outputs.Artifacts {
  if !art.HasLocation() {
+ log.Errorf("artifact has no location details:%#v", art)
  return errors.Errorf(errors.CodeBadRequest, "controller is not configured with a default archive location")
  }
  }

workflow/executor/executor.go

@@ -15,6 +15,7 @@ import (
  wfv1 "github.com/argoproj/argo/api/workflow/v1alpha1"
  "github.com/argoproj/argo/errors"
  artifact "github.com/argoproj/argo/workflow/artifacts"
+ "github.com/argoproj/argo/workflow/artifacts/artifactory"
  "github.com/argoproj/argo/workflow/artifacts/git"
  "github.com/argoproj/argo/workflow/artifacts/http"
  "github.com/argoproj/argo/workflow/artifacts/s3"
@@ -169,6 +170,10 @@ func (we *WorkflowExecutor) SaveArtifacts() error {
  shallowCopy := *we.Template.ArchiveLocation.S3
  art.S3 = &shallowCopy
  art.S3.Key = path.Join(art.S3.Key, fileName)
+ } else if we.Template.ArchiveLocation.Artifactory != nil {
+ shallowCopy := *we.Template.ArchiveLocation.Artifactory
+ art.Artifactory = &shallowCopy
+ art.Artifactory.URL = path.Join(art.Artifactory.URL, fileName)
  } else {
  return errors.Errorf(errors.CodeBadRequest, "Unable to determine path to store %s. Archive location provided no information", art.Name)
  }
@@ -280,6 +285,24 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv
 
  return &gitDriver, nil
  }
+ if art.Artifactory != nil {
+ // Getting Kubernetes namespace from the environment variables
+ namespace := os.Getenv(common.EnvVarNamespace)
+ username, err := we.getSecrets(namespace, art.Artifactory.UsernameSecret.Name, art.Artifactory.UsernameSecret.Key)
+ if err != nil {
+ return nil, err
+ }
+ password, err := we.getSecrets(namespace, art.Artifactory.PasswordSecret.Name, art.Artifactory.PasswordSecret.Key)
+ if err != nil {
+ return nil, err
+ }
+ driver := artifactory.ArtifactoryArtifactDriver{
+ Username: username,
+ Password: password,
+ }
+ return &driver, nil
+
+ }
  return nil, errors.Errorf(errors.CodeBadRequest, "Unsupported artifact driver for %s", art.Name)
 }