Skip to content

Commit

Permalink
Separate test definitions from test execution so that providers can u…
Browse files Browse the repository at this point in the history 
…se the definitions in their tests (pulumi#1583)

We'd like to test providers against the examples in this repository as
part of their CI process, i.e., before they are released. In order to do
that, this PR separates the test suite in misc/test into test
definitions and test execution. The execution part then consists of a
single test method instead of one per example, which looks like this:

```go
	for _, examples := range definitions.GetTestsForTag("azure") {
		for _, example := range examples {
			run(t, example)
		}
	}
```

Due to the use of subtests (`r.Run()`) the results are still cleanly
reported and individual tests can be run.

Providers can then `"github.com/pulumi/examples/misc/test/definitions"`
and use a similar snippet to run the same tests, but with their own
configuration to use local dependencies and SDKs. See
pulumi/pulumi-azure-native#3065 for an example.

There is a tag-based API to retrieve desired test definitions by
programming language, provider, or both.

This PR is limited to Azure and Azure Native so far. Other providers can
be added as needed.
  • Loading branch information
@thomas11
thomas11 committed Feb 26, 2024
1 parent 549157f commit 9f097a6
Show file tree
Hide file tree
Showing 15 changed files with 601 additions and 433 deletions.
31 changes: 16 additions & 15 deletions misc/test/aws_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/pulumi/examples/misc/test/helpers"
"github.com/pulumi/pulumi/pkg/v3/testing/integration"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
"github.com/stretchr/testify/assert"
Expand All @@ -38,7 +39,7 @@ func TestAccAwsGoEks(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["url"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello Kubernetes bootcamp!")
})
},
Expand All @@ -63,7 +64,7 @@ func TestAccAwsGoS3FolderComponent(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["websiteUrl"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, world!")
})
},
Expand All @@ -79,7 +80,7 @@ func TestAccAwsGoWebserver(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["publicIp"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, World!")
})
},
Expand Down Expand Up @@ -108,7 +109,7 @@ func TestAccAwsJsContainers(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["frontendURL"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, Pulumi!")
})
},
Expand All @@ -122,7 +123,7 @@ func TestAccAwsJsS3FolderComponent(t *testing.T) {
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "..", "..", "aws-js-s3-folder-component"),
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
assertHTTPResult(t, stack.Outputs["websiteUrl"].(string), nil, func(body string) bool {
helpers.AssertHTTPResult(t, stack.Outputs["websiteUrl"].(string), nil, func(body string) bool {
return assert.Contains(t, body, "Hello, Pulumi!")
})
},
Expand All @@ -148,7 +149,7 @@ func TestAccAwsJsWebserver(t *testing.T) {
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "..", "..", "aws-js-webserver"),
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
assertHTTPHelloWorld(t, stack.Outputs["publicHostName"], nil)
helpers.AssertHTTPHelloWorld(t, stack.Outputs["publicHostName"], nil)
},
})

Expand All @@ -160,7 +161,7 @@ func TestAccAwsJsWebserverComponent(t *testing.T) {
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "..", "..", "aws-js-webserver-component"),
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
assertHTTPHelloWorld(t, stack.Outputs["webUrl"], nil)
helpers.AssertHTTPHelloWorld(t, stack.Outputs["webUrl"], nil)
},
})

Expand All @@ -174,7 +175,7 @@ func TestAccAwsApiGatewayPyRoutes(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["url"].(string)
assertHTTPResultWithRetry(t, endpoint+"lambda", nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint+"lambda", nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, API Gateway!")
})
},
Expand All @@ -190,7 +191,7 @@ func TestAccAwsApiGatewayTsRoutes(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["url"].(string)
assertHTTPResultWithRetry(t, endpoint+"lambda", nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint+"lambda", nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, API Gateway!")
})
},
Expand Down Expand Up @@ -227,7 +228,7 @@ func TestAccAwsGoAppSync(t *testing.T) {
// "x-api-key": key,
// }

// assertHTTPResultShapeWithRetry(t, finalURL, headersMap, maxWait, func(body string) bool {
// helpers.AssertHTTPResultShapeWithRetry(t, finalURL, headersMap, maxWait, func(body string) bool {
// return !strings.Contains(body, "AccessDeniedException")
// }, func(body string) bool {
// return assert.Contains(t, body, "FirstCorp")
Expand Down Expand Up @@ -282,7 +283,7 @@ func TestAccAwsPyWebserver(t *testing.T) {
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "..", "..", "aws-py-webserver"),
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
assertHTTPResult(t, "http:https://"+stack.Outputs["public_dns"].(string), nil, func(body string) bool {
helpers.AssertHTTPResult(t, "http:https://"+stack.Outputs["public_dns"].(string), nil, func(body string) bool {
return assert.Contains(t, body, "Hello, World!")
})
},
Expand Down Expand Up @@ -312,7 +313,7 @@ func TestAccAwsTsApiGateway(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["endpoint"].(string)
assertHTTPResultWithRetry(t, endpoint+"hello", nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint+"hello", nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "route")
})
},
Expand Down Expand Up @@ -350,7 +351,7 @@ func TestAccAwsTsContainers(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 15 * time.Minute
endpoint := stack.Outputs["frontendURL"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, Pulumi!")
})
},
Expand Down Expand Up @@ -433,7 +434,7 @@ func TestAccAwsTsEksHelloWorld(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["serviceHostname"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Welcome to nginx")
})
},
Expand All @@ -449,7 +450,7 @@ func TestAccAwsTsHelloFargate(t *testing.T) {
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["url"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
helpers.AssertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello World!")
})
},
Expand Down
Loading

0 comments on commit 9f097a6

Please sign in to comment.