Annotate pod with outputs. Properly handle tar/gz of artifacts

api/workflow/v1/types.go

@@ -255,6 +255,19 @@ func (in *Inputs) GetParameterByName(name string) *Parameter {
  return nil
 }
 
+func (out *Outputs) HasOutputs() bool {
+ if out.Result != nil {
+ return true
+ }
+ if len(out.Artifacts) > 0 {
+ return true
+ }
+ if len(out.Parameters) > 0 {
+ return true
+ }
+ return false
+}
+
 func (args *Arguments) GetArtifactByName(name string) *Artifact {
  for _, art := range args.Artifacts {
  if art.Name == name {

cmd/argoexec/commands/artifacts.go

@@ -138,8 +138,15 @@ func saveArtifacts(cmd *cobra.Command, args []string) {
  wfExecutor := initExecutor()
  err := wfExecutor.SaveArtifacts()
  if err != nil {
- log.Errorf("Error saving output artifacts, %+v", err)
- os.Exit(1)
+ log.Fatalf("Error saving output artifacts, %+v", err)
+ }
+ err = wfExecutor.CaptureScriptResult()
+ if err != nil {
+ log.Fatalf("Error capturing script output, %+v", err)
+ }
+ err = wfExecutor.AnnotateOutputs()
+ if err != nil {
+ log.Fatalf("Error annotating outputs, %+v", err)
  }
  os.Exit(0)
 }

cmd/argoexec/commands/root.go

@@ -62,8 +62,7 @@ func initExecutor() *executor.WorkflowExecutor {
  // Read template
  err := GetTemplateFromPodAnnotations(podAnnotationsPath, &wfTemplate)
  if err != nil {
- log.Errorf("Error getting template %v", err)
- os.Exit(1)
+ log.Fatalf("Error getting template %v", err)
  }
 
  // Initialize in-cluster Kubernetes client
@@ -76,9 +75,14 @@ func initExecutor() *executor.WorkflowExecutor {
  if err != nil {
  panic(err.Error())
  }
+ podName, ok := os.LookupEnv(common.EnvVarPodName)
+ if !ok {
+ log.Fatalf("Unable to determine pod name from environment variable %s", common.EnvVarPodName)
+ }
 
  // Initialize workflow executor
  wfExecutor := executor.WorkflowExecutor{
+ PodName: podName,
  Template: wfTemplate,
  ClientSet: clientset,
  }

examples/ci.yaml

@@ -4,6 +4,9 @@ metadata:
  generateName: ci-example-
 spec:
  entrypoint: ci-example
+ # a temporary volume, named workdir, will be used as a working
+ # directory for this workflow. This volume is passed around
+ # from step to step.
  volumeClaimTemplates:
  - metadata:
  name: workdir
@@ -12,13 +15,25 @@ spec:
  resources:
  requests:
  storage: 1Gi
+
  templates:
  - name: ci-example
  steps:
  - - name: build
  template: build-golang-example
+ # the test step expands into three parallel steps running
+ # different operating system images. each mounts the workdir
+ # and runs the compiled binary from the build step.
  - - name: test
  template: run-hello
+ arguments:
+ parameters:
+ - name: os-image
+ value: "{{item.image}}:{{item.tag}}"
+ withItems:
+ - { image: 'debian', tag: '9.1' }
+ - { image: 'alpine', tag: '3.6' }
+ - { image: 'ubuntu', tag: '17.10' }
 
  - name: build-golang-example
  inputs:
@@ -39,10 +54,17 @@ spec:
  mountPath: /go
 
  - name: run-hello
+ inputs:
+ parameters:
+ - name: os-image
  container:
- image: debian:9.1
+ image: "{{inputs.parameters.os-image}}"
  command: [sh, -c]
- args: ["/go/src/github.com/golang/example/hello/hello"]
+ args: ["
+ uname -a ;
+ cat /etc/os-release ;
+ /go/src/github.com/golang/example/hello/hello
+ "]
  volumeMounts:
  - name: workdir
  mountPath: /go

examples/input-artifact-s3.yaml

@@ -8,7 +8,7 @@ spec:
  - name: input-artifact-s3-example
  inputs:
  artifacts:
- - name: CODE
+ - name: code
  path: /src
  s3:
  endpoint: storage.googleapis.com

examples/nested-workflow.yaml

@@ -31,17 +31,17 @@ spec:
  artifacts:
  - name: message
  steps:
- - PRINT:
+ - - name: print
  template: print-message
  arguments:
  artifacts:
- - name: message
+ - name: message2
  from: "{{inputs.artifacts.message}}"
 
  - name: print-message
  inputs:
  artifacts:
- - name: message
+ - name: message2
  path: /tmp/message
  container:
  image: alpine:latest

examples/sidecar-dind.yaml

@@ -20,6 +20,6 @@ spec:
  privileged: true
  # mirrorVolumeMounts will mount the same volumes specified in the main container
  # to the sidecar (including artifacts), at the same mountPaths. This enables
- # dind daemon to partially see the same filesystem as the main container in
- # order to use features such as docker volume binding
+ # dind daemon to (partially) see the same filesystem as the main container in
+ # order to use features such as docker volume binding.
  mirrorVolumeMounts: true

workflow/controller/operator.go

@@ -500,11 +500,12 @@ func (woc *wfOperationCtx) resolveReferences(stepGroup []wfv1.WorkflowStep, scop
  if art.From == "" {
  continue
  }
- art, err := scope.resolveArtifact(art.From)
+ resolvedArt, err := scope.resolveArtifact(art.From)
  if err != nil {
  return nil, err
  }
- newStep.Arguments.Artifacts[j] = *art
+ resolvedArt.Name = art.Name
+ newStep.Arguments.Artifacts[j] = *resolvedArt
  }
 
  newStepGroup[i] = newStep

workflow/controller/workflowpod.go

@@ -477,15 +477,6 @@ func addScriptVolume(pod *apiv1.Pod) {
  found = true
  break
  }
- if ctr.Name == common.WaitContainerName {
- // Also annotate the pod with output result
- ctr.Args = []string{ctr.Args[0] + `
- output=$(grep stdout /var/lib/docker/containers/$container_id/*.log | jq -r '.log')
- outputjson={\"result\":\"$output\"}
- kubectl annotate pods $ARGO_POD_NAME --overwrite workflows.argoproj.io/outputs=${outputjson}
- `}
- pod.Spec.Containers[i] = ctr
- }
  }
  if !found {
  panic("Unable to locate main container")