Skip to content

Commit

Permalink
Update providers during perf measure (pulumi#1576)
Browse files Browse the repository at this point in the history
For Python and TypeScript, provider references are floating, but for Go
they are always pinned. As a result TestAccAwsGoS3Folder was always
testing the performance of aws v6.0.2 not the later versions that
shipped some performance improvements. Compensate for this in the
benchmark by updating to latest.
  • Loading branch information
t0yv0 committed Feb 6, 2024
1 parent 7a0dbf4 commit 2bdfd3c
Showing 1 changed file with 14 additions and 2 deletions.
16 changes: 14 additions & 2 deletions misc/test/performance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import (
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/pulumi/pulumi-trace-tool/traces"
"github.com/pulumi/pulumi/pkg/v3/testing/integration"
"github.com/stretchr/testify/assert"
)

func bench(name, provider, runtime, lang string) traces.Benchmark {
Expand All @@ -28,15 +30,25 @@ func bench(name, provider, runtime, lang string) traces.Benchmark {

func TestAccAwsGoS3Folder(t *testing.T) {
benchmark := bench("aws-go-s3-folder", "aws", "go", "go")
dir := path.Join(getCwd(t), "..", "..", benchmark.Name)

// Update AWS to latest.
cmd := exec.Command("go", "get", "-u", "github.com/pulumi/pulumi-aws/sdk/v6")
cmd.Dir = dir
out, err := cmd.CombinedOutput()
require.NoErrorf(t, err, "Failed to update pulumi-aws to latest version: %s", out)
t.Logf("Updated pulumi-aws dependency to the latest version")

opts := integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "..", "..", benchmark.Name),
Dir: dir,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
maxWait := 10 * time.Minute
endpoint := stack.Outputs["websiteUrl"].(string)
assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
return assert.Contains(t, body, "Hello, world!")
})
},

}
test := getAWSBase(t).With(opts)
programTestAsBenchmark(t, benchmark, test)
Expand Down

0 comments on commit 2bdfd3c

Please sign in to comment.