Integrate trace collection into the perf bench (pulumi#1020)

.github/workflows/performance_metrics_cron.yml

@@ -1,3 +1,8 @@
+name: Run Performance Metrics Cron Job
+"on":
+ workflow_dispatch: {}
+ schedule:
+ - cron: 0 9 * * *
 env:
  ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
  ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
@@ -25,6 +30,10 @@ jobs:
  name: performance_metrics_cron
  runs-on: ${{ matrix.platform }}
  steps:
+ - name: Install Go ${{ matrix.go-version }}
+ uses: actions/setup-go@v1
+ with:
+ go-version: ${{matrix.go-version}}
  - name: Install DotNet ${{ matrix.dotnet-version }}
  uses: actions/setup-dotnet@v1
  with:
@@ -37,10 +46,6 @@ jobs:
  uses: actions/setup-python@v1
  with:
  python-version: ${{matrix.python-version}}
- - name: Install Go ${{ matrix.go-version }}
- uses: actions/setup-go@v1
- with:
- go-version: ${{matrix.go-version}}
  - name: Install Python Deps
  run: |-
  pip3 install virtualenv==20.0.23
@@ -55,16 +60,32 @@ jobs:
  role-session-name: examples@githubActions
  role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
  - uses: actions/checkout@v2
- - name: Install Latest Stable Pulumi CLI
- uses: pulumi/setup-pulumi@v2
+ - name: "Temporarily hack go.mod to get pre-release pulumi deps"
+ run: ./misc/test/temp_hack_go_mod.sh
+ # - name: Install Latest Stable Pulumi CLI
+ # uses: pulumi/setup-pulumi@v2
+ - name: Install Pulumi CLI with tracing additions (temp until Latest Stable has it)
+ run: |-
+ curl -sSL https://get.pulumi.com | bash -s -- --version 3.5.0-alpha.1623852893
+ echo "/home/runner/.pulumi/bin" >> $GITHUB_PATH
  - run: echo "Currently Pulumi $(pulumi version) is installed"
  - name: Install Testing Dependencies
  run: make ensure
+ - name: Create a Folder for Collecting Traces
+ run: |-
+ mkdir "$PWD/traces"
  - name: Add Test Trace Details
  run: |
- echo "PULUMI_VERSION=$(pulumi version)" >> $GITHUB_ENV
- - name: Running Performance Matrix
+ echo "PULUMI_VERSION=$(pulumi version)" >> $GITHUB_ENV
+ echo "PULUMI_TRACING_TAG_PULUMI_VERSION=$(pulumi version)" >> $GITHUB_ENV
+ echo "PULUMI_TRACING_DIR=$PWD/traces" >> $GITHUB_ENV
+ - name: Run Performance Matrix
  run: make performance_test_set
+ - name: Upload Traces to s3:https://cli-performance-metrics
+ run: |-
+ gzip -9 "$PWD/traces/metrics.csv"
+ RAND=$(python -c 'import uuid; print(uuid.uuid4())')
+ aws s3 cp "$PWD/traces/metrics.csv.gz" "s3:https://cli-performance-metrics/$(date -u +'%Y-%m-%d')/cli_performance_metrics_$RAND.csv.gz"
  strategy:
  fail-fast: false
  matrix:
@@ -78,8 +99,3 @@ jobs:
  - ubuntu-latest
  python-version:
  - "3.7"
-name: Run Performance Metrics Cron Job
-"on":
- workflow_dispatch: {}
- schedule:
- - cron: 0 9 * * *

misc/test/go.mod

@@ -3,10 +3,11 @@ module github.com/pulumi/examples
 go 1.16
 
 require (
- github.com/aws/aws-sdk-go v1.37.31
+ github.com/aws/aws-sdk-go v1.38.35
  github.com/mitchellh/go-homedir v1.1.0
+ github.com/pulumi/pulumi-trace-tool v0.0.0-20210616213220-f618784d3cfe
  github.com/pulumi/pulumi/pkg/v3 v3.0.0
- github.com/pulumi/pulumi/sdk/v3 v3.0.0
+ github.com/pulumi/pulumi/sdk/v3 v3.3.1
  github.com/stretchr/testify v1.6.1
  gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
  gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect

misc/test/go.sum

@@ -117,8 +117,9 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.37.31 h1:eK7hgg1H4xivwopAbnzfQ7ZBbDb9cEkGDivd9rUMnJs=
 github.com/aws/aws-sdk-go v1.37.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.38.35 h1:7AlAO0FC+8nFjxiGKEmq0QLpiA8/XFr6eIxgRTwkdTg=
+github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -212,8 +213,9 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -369,6 +371,7 @@ github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -463,10 +466,13 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/pulumi/pulumi-trace-tool v0.0.0-20210616213220-f618784d3cfe h1:HwAfD2D0S7j1W0LPQTwJJHh3pet1Ng93Zx1bEePyer8=
+github.com/pulumi/pulumi-trace-tool v0.0.0-20210616213220-f618784d3cfe/go.mod h1:uZ1V56leSYGxC0MDhYSviag6D+CPiopeCyM6rRNBpv4=
 github.com/pulumi/pulumi/pkg/v3 v3.0.0 h1:Wj920eXTkNydNXHX7YQESO2vSuW33X+n1BS4Uywstzs=
 github.com/pulumi/pulumi/pkg/v3 v3.0.0/go.mod h1:XsgC0PpPdjzZAj9YyEMvtU0ei542zOGxS7M1EzFmH84=
-github.com/pulumi/pulumi/sdk/v3 v3.0.0 h1:zkragE05t1Rco/ymfqMU4UXdEmKMmOH0SXhUKQxujxQ=
 github.com/pulumi/pulumi/sdk/v3 v3.0.0/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds=
+github.com/pulumi/pulumi/sdk/v3 v3.3.1 h1:E64NheQ5X2fz3DcQsZ7lDV7js51GNFPnXiXWfPCACnE=
+github.com/pulumi/pulumi/sdk/v3 v3.3.1/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds=
 github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
 github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -771,6 +777,7 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200608174601-1b747fd94509/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
@@ -781,8 +788,9 @@ golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4X
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516 h1:E8xavSjXY8LFvcMSu/8Fjztt+SerwKnuAUOdS+aCXUM=
 golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

misc/test/performance_test.go

@@ -3,86 +3,182 @@
 package test
 
 import (
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
  "path"
+ "path/filepath"
+ "strings"
  "testing"
  "time"
 
+ "github.com/pulumi/pulumi-trace-tool/traces"
  "github.com/pulumi/pulumi/pkg/v3/testing/integration"
  "github.com/stretchr/testify/assert"
 )
 
-func TestAccAwsGoS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-go-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- maxWait := 10 * time.Minute
- endpoint := stack.Outputs["websiteUrl"].(string)
- assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
- return assert.Contains(t, body, "Hello, world!")
- })
- },
- })
+type bench struct {
+ name string
+ provider string
+ runtime string
+ language string
+}
 
+func TestAccAwsGoS3Folder(t *testing.T) {
+ benchmark := bench{"aws-go-s3-folder", "aws", "go", "go"}
+ opts := integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", benchmark.name),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ maxWait := 10 * time.Minute
+ endpoint := stack.Outputs["websiteUrl"].(string)
+ assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
+ return assert.Contains(t, body, "Hello, world!")
+ })
+ },
+ }
+ test := getAWSBase(t).With(opts).With(tracingOpts(t, benchmark))
  integration.ProgramTest(t, &test)
 }
 
 func TestAccAwsCsS3Folder(t *testing.T) {
- test := getAWSBase(t).
-  With(integration.ProgramTestOptions{
-  Dir: path.Join(getCwd(t), "..", "..", "aws-cs-s3-folder"),
-  ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
-  maxWait := 10 * time.Minute
-  endpoint := stack.Outputs["Endpoint"].(string)
-  assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
-  return assert.Contains(t, body, "Hello, world!")
-  })
-  },
-  })
-
+ benchmark := bench{"aws-cs-s3-folder", "aws", "dotnet", "csharp"}
+ opts := integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", benchmark.name),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ maxWait := 10 * time.Minute
+ endpoint := stack.Outputs["Endpoint"].(string)
+ assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
+ return assert.Contains(t, body, "Hello, world!")
+ })
+ },
+ }
+ test := getAWSBase(t).With(opts).With(tracingOpts(t, benchmark))
  integration.ProgramTest(t, &test)
 }
 
 func TestAccAwsFsS3Folder(t *testing.T) {
- test := getAWSBase(t).
-  With(integration.ProgramTestOptions{
-  Dir: path.Join(getCwd(t), "..", "..", "aws-fs-s3-folder"),
-  ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
-  maxWait := 10 * time.Minute
-  endpoint := stack.Outputs["endpoint"].(string)
-  assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
-  return assert.Contains(t, body, "Hello, world!")
-  })
-  },
-  })
-
+ benchmark := bench{"aws-fs-s3-folder", "aws", "dotnet", "fsharp"}
+ opts := integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", benchmark.name),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ maxWait := 10 * time.Minute
+ endpoint := stack.Outputs["endpoint"].(string)
+ assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
+ return assert.Contains(t, body, "Hello, world!")
+ })
+ },
+ }
+ test := getAWSBase(t).With(opts).With(tracingOpts(t, benchmark))
  integration.ProgramTest(t, &test)
 }
 
 func TestAccAwsJsS3Folder(t *testing.T) {
- test := getAWSBase(t).
-  With(integration.ProgramTestOptions{
-  Dir: path.Join(getCwd(t), "..", "..", "aws-js-s3-folder"),
-  ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
-  assertHTTPResult(t, "http:https://"+stack.Outputs["websiteUrl"].(string), nil, func(body string) bool {
-  return assert.Contains(t, body, "Hello, Pulumi!")
-  })
-  },
-  })
-
+ benchmark := bench{"aws-js-s3-folder", "aws", "nodejs", "js"}
+ opts := integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", benchmark.name),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ assertHTTPResult(t, "http:https://"+stack.Outputs["websiteUrl"].(string), nil, func(body string) bool {
+ return assert.Contains(t, body, "Hello, Pulumi!")
+ })
+ },
+ }
+ test := getAWSBase(t).With(opts).With(tracingOpts(t, benchmark))
  integration.ProgramTest(t, &test)
 }
 
 func TestAccAwsPyS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-py-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- assertHTTPResult(t, "http:https://"+stack.Outputs["website_url"].(string), nil, func(body string) bool {
- return assert.Contains(t, body, "Hello, Pulumi!")
- })
+ benchmark := bench{"aws-py-s3-folder", "aws", "python", "python"}
+ opts := integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", benchmark.name),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ assertHTTPResult(t, "http:https://"+stack.Outputs["website_url"].(string), nil, func(body string) bool {
+ return assert.Contains(t, body, "Hello, Pulumi!")
+ })
+ },
+ }
+ test := getAWSBase(t).With(opts).With(tracingOpts(t, benchmark))
+ integration.ProgramTest(t, &test)
+}
+
+func TestMain(m *testing.M) {
+ code := m.Run()
+
+ dir := tracingDir()
+ if dir != "" {
+ // After all tests run with tracing, compute metrics
+ // on the entire set.
+ err := computeMetrics(dir)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ os.Exit(code)
+}
+
+func tracingDir() string {
+ return os.Getenv("PULUMI_TRACING_DIR")
+}
+
+func tracingOpts(t *testing.T, benchmark bench) integration.ProgramTestOptions {
+ dir := tracingDir()
+
+ if dir != "" {
+ return integration.ProgramTestOptions{
+ Env: []string{
+ "PULUMI_TRACING_TAG_REPO=pulumi/examples",
+ fmt.Sprintf("PULUMI_TRACING_TAG_BENCHMARK_NAME=%s", benchmark.name),
+ fmt.Sprintf("PULUMI_TRACING_TAG_BENCHMARK_PROVIDER=%s", benchmark.provider),
+ fmt.Sprintf("PULUMI_TRACING_TAG_BENCHMARK_RUNTIME=%s", benchmark.runtime),
+ fmt.Sprintf("PULUMI_TRACING_TAG_BENCHMARK_LANGUAGE=%s", benchmark.language),
+ "PULUMI_TRACING_MEMSTATS_POLL_INTERVAL=100ms",
  },
- })
+ Tracing: fmt.Sprintf("file:%s",
+ filepath.Join(dir, fmt.Sprintf("%s-{command}.trace", benchmark.name))),
+ }
+ }
 
- integration.ProgramTest(t, &test)
+ return integration.ProgramTestOptions{}
+}
+
+func computeMetrics(dir string) error {
+ cwd, err := os.Getwd()
+ if err != nil {
+ return err
+ }
+
+ defer os.Chdir(cwd)
+
+ err = os.Chdir(dir)
+ if err != nil {
+ return err
+ }
+
+ files, err := ioutil.ReadDir(".")
+ if err != nil {
+ return err
+ }
+
+ var traceFiles []string
+ for _, f := range files {
+ if strings.HasSuffix(f.Name(), ".trace") {
+ traceFiles = append(traceFiles, f.Name())
+ }
+ }
+
+ if err := traces.ToCsv(traceFiles, "traces.csv", "filename"); err != nil {
+ return err
+ }
+
+ f, err := os.Create("metrics.csv")
+ if err != nil {
+ return err
+ }
+
+ if err := traces.Metrics("traces.csv", "filename", f); err != nil {
+ return err
+ }
+
+ return nil
 }