feat(executor): Auto create s3 bucket if not present. Closes argoproj…

…#3586  (argoproj#4574)

Signed-off-by: Alex Capras <[email protected]>

api/jsonschema/schema.json

@@ -438,6 +438,16 @@
  },
  "type": "object"
  },
+ "io.argoproj.workflow.v1alpha1.CreateS3BucketOptions": {
+ "description": "CreateS3BucketOptions options used to determine automatic automatic bucket-creation process",
+ "properties": {
+ "objectLocking": {
+ "description": "ObjectLocking Enable object locking",
+ "type": "boolean"
+ }
+ },
+ "type": "object"
+ },
  "io.argoproj.workflow.v1alpha1.CronWorkflow": {
  "description": "CronWorkflow is the definition of a scheduled workflow resource",
  "properties": {
@@ -1530,6 +1540,10 @@
  "description": "Bucket is the name of the bucket",
  "type": "string"
  },
+ "createBucketIfNotPresent": {
+ "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.CreateS3BucketOptions",
+ "description": "CreateBucketIfNotPresent tells the driver to attempt to create the S3 bucket for output artifacts, if it doesn't exist"
+ },
  "endpoint": {
  "description": "Endpoint is the hostname of the bucket endpoint",
  "type": "string"

api/openapi-spec/swagger.json

@@ -2492,6 +2492,16 @@
  }
  }
  },
+ "io.argoproj.workflow.v1alpha1.CreateS3BucketOptions": {
+ "description": "CreateS3BucketOptions options used to determine automatic automatic bucket-creation process",
+ "type": "object",
+ "properties": {
+ "objectLocking": {
+ "description": "ObjectLocking Enable object locking",
+ "type": "boolean"
+ }
+ }
+ },
  "io.argoproj.workflow.v1alpha1.CronWorkflow": {
  "description": "CronWorkflow is the definition of a scheduled workflow resource",
  "type": "object",
@@ -3590,6 +3600,10 @@
  "description": "Bucket is the name of the bucket",
  "type": "string"
  },
+ "createBucketIfNotPresent": {
+ "description": "CreateBucketIfNotPresent tells the driver to attempt to create the S3 bucket for output artifacts, if it doesn't exist",
+ "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.CreateS3BucketOptions"
+ },
  "endpoint": {
  "description": "Endpoint is the hostname of the bucket endpoint",
  "type": "string"

docs/fields.md

@@ -3606,6 +3606,7 @@ S3Artifact is the location of an S3 artifact
 |:----------:|:----------:|---------------|
 |`accessKeySecret`|[`SecretKeySelector`](#secretkeyselector)|AccessKeySecret is the secret selector to the bucket's access key|
 |`bucket`|`string`|Bucket is the name of the bucket|
+|`createBucketIfNotPresent`|[`CreateS3BucketOptions`](#creates3bucketoptions)|CreateBucketIfNotPresent tells the driver to attempt to create the S3 bucket for output artifacts, if it doesn't exist|
 |`endpoint`|`string`|Endpoint is the hostname of the bucket endpoint|
 |`insecure`|`boolean`|Insecure will connect to the service with TLS|
 |`key`|`string`|Key is the key in the bucket where the artifact resides|
@@ -3982,6 +3983,15 @@ Header indicate a key-value request header to be used when fetching artifacts ov
 |`name`|`string`|Name is the header name|
 |`value`|`string`|Value is the literal value to use for the header|
 
+## CreateS3BucketOptions
+
+CreateS3BucketOptions options used to determine automatic automatic bucket-creation process
+
+### Fields
+| Field Name | Field Type | Description |
+|:----------:|:----------:|---------------|
+|`objectLocking`|`boolean`|ObjectLocking Enable object locking|
+
 ## SuppliedValueFrom
 
 SuppliedValueFrom is a placeholder for a value to be filled in directly, either through the CLI, API, etc.

go.mod

@@ -9,7 +9,7 @@ require (
  github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
  github.com/aliyun/aliyun-oss-go-sdk v2.0.6+incompatible
  github.com/antonmedv/expr v1.8.2
- github.com/argoproj/pkg v0.2.0
+ github.com/argoproj/pkg v0.3.0
  github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/blushft/go-diagrams v0.0.0-20201006005127-c78c821223d9
@@ -37,6 +37,7 @@ require (
  github.com/klauspost/compress v1.9.7 // indirect
  github.com/lib/pq v1.3.0 // indirect
  github.com/mattn/goreman v0.3.5
+ github.com/minio/minio-go/v7 v7.0.2
  github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
  github.com/pkg/errors v0.9.1
  github.com/prometheus/client_golang v1.0.0

go.sum

@@ -83,8 +83,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
 github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4=
 github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
-github.com/argoproj/pkg v0.2.0 h1:ETgC600kr8WcAi3MEVY5sA1H7H/u1/IysYOobwsZ8No=
-github.com/argoproj/pkg v0.2.0/go.mod h1:F4TZgInLUEjzsWFB/BTJBsewoEy0ucnKSq6vmQiD/yc=
+github.com/argoproj/pkg v0.3.0 h1:0xxNHc9duXZt1TlDVVsBzcGm8S/V5Bc3eNqawz3wFo8=
+github.com/argoproj/pkg v0.3.0/go.mod h1:F4TZgInLUEjzsWFB/BTJBsewoEy0ucnKSq6vmQiD/yc=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=

manifests/base/crds/full/argoproj.io_clusterworkflowtemplates.yaml

@@ -559,6 +559,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -1421,6 +1426,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -1721,6 +1731,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -2610,6 +2625,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -3571,6 +3591,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -4017,6 +4042,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:

manifests/base/crds/full/argoproj.io_cronworkflows.yaml

@@ -580,6 +580,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -1442,6 +1447,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -1742,6 +1752,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -2631,6 +2646,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -3592,6 +3612,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:
@@ -4038,6 +4063,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure:

manifests/base/crds/full/argoproj.io_workfloweventbindings.yaml

@@ -302,6 +302,11 @@ spec:
  type: object
  bucket:
  type: string
+ createBucketIfNotPresent:
+ properties:
+ objectLocking:
+ type: boolean
+ type: object
  endpoint:
  type: string
  insecure: