feat(archive): allow specifying a compression level (argoproj#2575)

api/openapi-spec/swagger.json

@@ -2838,7 +2838,14 @@
  },
  "io.argoproj.workflow.v1alpha1.TarStrategy": {
  "type": "object",
- "title": "TarStrategy will tar and gzip the file or directory when saving"
+ "title": "TarStrategy will tar and gzip the file or directory when saving",
+ "properties": {
+ "compressionLevel": {
+ "description": "CompressionLevel specifies the gzip compression level to use for the artifact.\nDefaults to gzip.DefaultCompression.",
+ "type": "integer",
+ "format": "int32"
+ }
+ }
  },
  "io.argoproj.workflow.v1alpha1.Template": {
  "type": "object",

examples/README.md

@@ -372,6 +372,36 @@ spec:
 The `whalesay` template uses the `cowsay` command to generate a file named `/tmp/hello-world.txt`. It then `outputs` this file as an artifact named `hello-art`. In general, the artifact's `path` may be a directory rather than just a file. The `print-message` template takes an input artifact named `message`, unpacks it at the `path` named `/tmp/message` and then prints the contents of `/tmp/message` using the `cat` command.
 The `artifact-example` template passes the `hello-art` artifact generated as an output of the `generate-artifact` step as the `message` input artifact to the `print-message` step. DAG templates use the tasks prefix to refer to another task, for example `{{tasks.generate-artifact.outputs.artifacts.hello-art}}`.
 
+Artifacts are packaged as Tarballs and gzipped by default. You may customize this behavior by specifying an archive strategy, using the `archive` field. For example:
+
+```yaml
+<... snipped ...>
+ outputs:
+ artifacts:
+ # default behavior - tar+gzip default compression.
+ - name: hello-art-1
+ path: /tmp/hello_world.txt
+
+ # disable archiving entirely - upload the file / directory as is.
+ # this is useful when the container layout matches the desired target repository layout. 
+ - name: hello-art-2
+ path: /tmp/hello_world.txt
+ archive:
+ none: {}
+
+ # customize the compression behavior (disabling it here).
+ # this is useful for files with varying compression benefits, 
+ # e.g. disabling compression for a cached build workspace and large binaries, 
+ # or increasing compression for "perfect" textual data - like a json/xml export of a large database.
+ - name: hello-art-3
+ path: /tmp/hello_world.txt
+ archive:
+ tar:
+ # no compression (also accepts the standard gzip 1 to 9 values)
+ compressionLevel: 0
+<... snipped ...>
+``` 
+
 ## The Structure of Workflow Specs
 
 We now know enough about the basic components of a workflow spec to review its basic structure:

examples/artifact-disable-archive.yaml

@@ -1,6 +1,8 @@
 # This example demonstrates the ability to disable the default behavior of archiving (tar.gz)
 # when saving output artifacts. For directories, when archive is set to none, files in directory
 # will be copied recursively in the case of S3.
+# Another option is to keep the archiving behavior, but skip or modify the compression
+# behavior using the 'tar.compressionLevel' field.
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
@@ -20,12 +22,14 @@ spec:
  from: "{{steps.generate-artifact.outputs.artifacts.etc}}"
  - name: hello-txt
  from: "{{steps.generate-artifact.outputs.artifacts.hello-txt}}"
+ - name: hello-txt-nc
+ from: "{{steps.generate-artifact.outputs.artifacts.hello-txt-nc}}"
 
  - name: whalesay
  container:
  image: docker/whalesay:latest
  command: [sh, -c]
- args: ["cowsay hello world | tee /tmp/hello_world.txt ; sleep 1"]
+ args: ["cowsay hello world | tee /tmp/hello_world.txt | tee /tmp/hello_world_nc.txt ; sleep 1"]
  outputs:
  artifacts:
  - name: etc
@@ -36,6 +40,12 @@ spec:
  path: /tmp/hello_world.txt
  archive:
  none: {}
+ - name: hello-txt-nc
+ path: /tmp/hello_world_nc.txt
+ archive:
+ tar:
+ # no compression (also accepts the standard gzip 1 to 9 values)
+ compressionLevel: 0
 
  - name: print-message
  inputs:
@@ -44,8 +54,10 @@ spec:
  path: /tmp/etc
  - name: hello-txt
  path: /tmp/hello.txt
+ - name: hello-txt-nc
+ path: /tmp/hello_nc.txt
  container:
  image: alpine:latest
  command: [sh, -c]
  args:
- - cat /tmp/hello.txt && cd /tmp/etc && find .
+ - cat /tmp/hello.txt && cat /tmp/hello_nc.txt && cd /tmp/etc && find .

pkg/apiclient/clusterworkflowtemplate/cluster-workflow-template.swagger.json

@@ -1252,6 +1252,13 @@
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.TarStrategy": {
  "type": "object",
+ "properties": {
+ "compressionLevel": {
+ "type": "integer",
+ "format": "int32",
+ "description": "CompressionLevel specifies the gzip compression level to use for the artifact.\nDefaults to gzip.DefaultCompression."
+ }
+ },
  "title": "TarStrategy will tar and gzip the file or directory when saving"
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Template": {

pkg/apiclient/cronworkflow/cron-workflow.swagger.json

@@ -1357,6 +1357,13 @@
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.TarStrategy": {
  "type": "object",
+ "properties": {
+ "compressionLevel": {
+ "type": "integer",
+ "format": "int32",
+ "description": "CompressionLevel specifies the gzip compression level to use for the artifact.\nDefaults to gzip.DefaultCompression."
+ }
+ },
  "title": "TarStrategy will tar and gzip the file or directory when saving"
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Template": {

pkg/apiclient/workflow/workflow.swagger.json

@@ -1723,6 +1723,13 @@
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.TarStrategy": {
  "type": "object",
+ "properties": {
+ "compressionLevel": {
+ "type": "integer",
+ "format": "int32",
+ "description": "CompressionLevel specifies the gzip compression level to use for the artifact.\nDefaults to gzip.DefaultCompression."
+ }
+ },
  "title": "TarStrategy will tar and gzip the file or directory when saving"
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Template": {

pkg/apiclient/workflowarchive/workflow-archive.swagger.json

@@ -1154,6 +1154,13 @@
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.TarStrategy": {
  "type": "object",
+ "properties": {
+ "compressionLevel": {
+ "type": "integer",
+ "format": "int32",
+ "description": "CompressionLevel specifies the gzip compression level to use for the artifact.\nDefaults to gzip.DefaultCompression."
+ }
+ },
  "title": "TarStrategy will tar and gzip the file or directory when saving"
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Template": {

pkg/apiclient/workflowtemplate/workflow-template.swagger.json

@@ -1225,6 +1225,13 @@
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.TarStrategy": {
  "type": "object",
+ "properties": {
+ "compressionLevel": {
+ "type": "integer",
+ "format": "int32",
+ "description": "CompressionLevel specifies the gzip compression level to use for the artifact.\nDefaults to gzip.DefaultCompression."
+ }
+ },
  "title": "TarStrategy will tar and gzip the file or directory when saving"
  },
  "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Template": {