More robust perf matrix (pulumi#1265)

* Add helper Make targets for testing individual examples

* Use gotestfmt, tolerate failure, upload traces artifact

* Update trace-tool dep to tolerate CSV parse issues

* Add GITHUB_TOKEN

* Install known version

* Add GITHUB_TOKE

* Fix syntax

* Forgot -json switch

.github/workflows/performance_metrics_cron.yml

@@ -65,6 +65,14 @@ jobs:
  - run: echo "Currently Pulumi $(pulumi version) is installed"
  - name: Install Testing Dependencies
  run: make ensure
+ - name: Install gotestfmt
+ uses: jaxxstorm/[email protected]
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ repo: haveyoudebuggedit/gotestfmt
+ tag: v2.3.2
+ cache: enable
  - name: Create a Folder for Collecting Traces
  run: |-
  mkdir "$PWD/traces"
@@ -74,7 +82,19 @@ jobs:
  echo "PULUMI_TRACING_TAG_PULUMI_VERSION=$(pulumi version)" >> $GITHUB_ENV
  echo "PULUMI_TRACING_DIR=$PWD/traces" >> $GITHUB_ENV
  - name: Run Performance Matrix
- run: make performance_test_set
+ # Even if some Go tests fail, others may have succeded and
+ # produced perf data; continue to try to upload it.
+ continue-on-error: true
+ shell: bash
+ run: |
+ set -euo pipefail
+ cd misc/test && go test . --timeout 4h -count=1 -json -short -parallel 40 --tags=Performance 2>&1 | tee /tmp/gotest.log | gotestfmt
+ - name: Upload traces GHA artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: traces
+ path: traces
+ retention-days: 14
  - name: Upload Traces to s3:https://cli-performance-metrics
  run: |-
  F="$PWD/traces/metrics.parquet.snappy"

Makefile

@@ -34,3 +34,18 @@ destroy_test_infra:
  echo "Tearing down test infra"
  ./misc/scripts/destroy-ci-cluster.sh $(StackName)
 
+# Run a test of a single example. Example usage:
+#
+# make test_example.TestAccAwsPyS3Folder
+test_example.%:
+ cd misc/test && go test -test.v -run "^$*$$" -tags all
+
+# Some of the examples double up as performance benchmarks. Run:
+#
+# make bench_example.TestAccAwsPyS3Folder
+#
+# This will run the example and populate ./traces with performance data.
+# See also https://www.pulumi.com/docs/support/troubleshooting/#performance
+bench_example.%:
+ mkdir -p ./traces
+ cd misc/test && PULUMI_TRACING_DIR=${PWD}/traces go test -test.v -run "^$*$$" -tags all

misc/test/go.mod

@@ -5,7 +5,7 @@ go 1.16
 require (
  github.com/aws/aws-sdk-go v1.38.35
  github.com/mitchellh/go-homedir v1.1.0
- github.com/pulumi/pulumi-trace-tool v0.0.0-20210629202931-10ae87ca9439
+ github.com/pulumi/pulumi-trace-tool v0.0.0-20220818154825-5db04013ec98
  github.com/pulumi/pulumi/pkg/v3 v3.5.1
  github.com/pulumi/pulumi/sdk/v3 v3.5.1
  github.com/stretchr/testify v1.7.0

misc/test/go.sum

@@ -573,8 +573,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/pulumi/pulumi-trace-tool v0.0.0-20210629202931-10ae87ca9439 h1:8nwCPl0rSQOxRYWrr9u6ATVMdX/YWF1hcn3UyHHQ1mo=
-github.com/pulumi/pulumi-trace-tool v0.0.0-20210629202931-10ae87ca9439/go.mod h1:CrD+qPXZjdh0T0+DK5Xclbr1eKKWnQt6svOOeTh4pl4=
+github.com/pulumi/pulumi-trace-tool v0.0.0-20220818154825-5db04013ec98 h1:sVQ6w7xAYPs9126IAHlGdj5QIx8QHse7rEv9kRp54sg=
+github.com/pulumi/pulumi-trace-tool v0.0.0-20220818154825-5db04013ec98/go.mod h1:CrD+qPXZjdh0T0+DK5Xclbr1eKKWnQt6svOOeTh4pl4=
 github.com/pulumi/pulumi/pkg/v3 v3.5.1 h1:6+Y5Fot9mNRxNK23eT9HTeHt0bnjzpqtuJDAhK3re10=
 github.com/pulumi/pulumi/pkg/v3 v3.5.1/go.mod h1:aAGoadWl60wVSE8Ig2FqcxUdfrmMKV6xfErcTOToIV4=
 github.com/pulumi/pulumi/sdk/v3 v3.3.1/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds=