Tooling for integration tests (open-telemetry#321)

This adds make and circleci config needed for running separate integration
tests. These tests are intended to still be run from go test so that they can
output coverage data but they may be longer running and talk to external
services like docker.

.circleci/config.yml

@@ -1,10 +1,10 @@
 version: 2.1
 parameters:
  run-build-publish:
- type: boolean 
- default: true 
+ type: boolean
+ default: true
  run-stability-tests:
- type: boolean 
+ type: boolean
  default: false
  collector-sha:
  type: string
@@ -14,8 +14,38 @@ executors:
  golang:
  docker:
  - image: cimg/go:1.14
+ machine:
+ machine:
+ image: ubuntu-1604:201903-01
 
 commands:
+ setup:
+ steps:
+ - checkout
+ - restore_module_cache
+ - run:
+ name: Install deps
+ command: make -j8 for-all-target TARGET=dep
+ - run:
+ name: Install tools
+ command: make install-tools
+ - run:
+ name: Install testbed tools
+ command: make -C testbed install-tools
+ - save_module_cache
+
+ setup_go:
+ steps:
+ - run:
+ name: Install Go 1.14
+ command: |
+ sudo rm -rf /usr/local/go
+ curl -L https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz | sudo tar xz -C /usr/local
+ - run:
+ name: Add ~/go/bin to PATH
+ command: |
+ echo 'export PATH=$HOME/go/bin:$PATH' >> $BASH_ENV
+
  restore_workspace:
  steps:
  - attach_to_workspace
@@ -81,7 +111,7 @@ workflows:
  - build
  - publish-dev:
  requires:
- - run-stability-tests 
+ - run-stability-tests
 
  build-publish:
  when: << pipeline.parameters.run-build-publish >>
@@ -120,7 +150,7 @@ workflows:
  filters:
  tags:
  only: /.*/
- - test:
+ - unit-tests:
  requires:
  - setup
  filters:
@@ -129,7 +159,8 @@ workflows:
  - publish-stable:
  requires:
  - lint
- - test
+ - unit-tests
+ - integration-tests
  - build
  - cross-compile
  filters:
@@ -140,30 +171,24 @@ workflows:
  - spawn-stability-tests-job:
  requires:
  - lint
- - test
+ - unit-tests
+ - integration-tests
  - build
  filters:
  branches:
  only: /master|release\/.+/
  tags:
  ignore: /.*/
+ - integration-tests:
+ filters:
+ tags:
+ only: /.*/
 
 jobs:
  setup:
  executor: golang
  steps:
- - checkout
- - restore_module_cache
- - run:
- name: Install deps
- command: make -j8 for-all-target TARGET='dep'
- - run:
- name: Install tools
- command: make install-tools
- - run:
- name: Install testbed tools
- command: make -C testbed install-tools
- - save_module_cache
+ - setup
  - persist_to_workspace:
  root: ~/
  paths:
@@ -175,7 +200,7 @@ jobs:
  - restore_workspace
  - run:
  name: Lint
- command: make -j8 for-all-target TARGET="lint"
+ command: make -j8 for-all-target TARGET=lint
  - run:
  name: Checks
  command: make -j4 checklicense impi misspell
@@ -213,16 +238,16 @@ jobs:
  root: ~/
  paths: project/bin
 
- test:
+ unit-tests:
  executor: golang
  steps:
  - restore_workspace
  - run:
- name: Coverage tests
- command: make test-with-cover
+ name: Unit test coverage
+ command: make unit-tests-with-cover
  - run:
- name: Code coverage
- command: bash <(curl -s https://codecov.io/bash)
+ name: Upload unit test coverage
+ command: bash <(curl -s https://codecov.io/bash) -F unit
 
  loadtest:
  executor: golang
@@ -247,7 +272,7 @@ jobs:
  repo: opentelemetry-collector-contrib
  tag: ${CIRCLE_TAG:1}
  - run:
- name: Calculate checksums 
+ name: Calculate checksums
  command: cd bin && shasum -a 256 * > checksums.txt
  - run:
  name: Create Github release and upload artifacts
@@ -256,7 +281,7 @@ jobs:
  publish-dev:
  executor: golang
  steps:
- - restore_workspace 
+ - restore_workspace
  - setup_remote_docker
  - publish_docker_images:
  repo: opentelemetry-collector-contrib-dev
@@ -286,7 +311,7 @@ jobs:
  run-stability-tests:
  executor: golang
  steps:
- - restore_workspace 
+ - restore_workspace
  - run:
  name: Run stability tests
  command: make stability-tests
@@ -302,3 +327,24 @@ jobs:
  "body": "Link to failed job: '"${CIRCLE_BUILD_URL}"'."
  }'
  when: on_fail
+
+ integration-tests:
+ executor: machine
+ environment:
+ GOPATH: /home/circleci/go
+ steps:
+ - setup_go
+ - setup
+ - run:
+ name: Integration tests with coverage
+ command: |
+ mkdir -p test-results/junit
+ trap "go-junit-report -set-exit-code < test-results/go-integration-tests.out > test-results/junit/results.xml" EXIT
+ make integration-tests-with-cover | tee test-results/go-integration-tests.out
+ - run:
+ name: Upload integration test coverage
+ command: bash <(curl -s https://codecov.io/bash) -F integration
+ - store_test_results:
+ path: test-results/junit
+ - store_artifacts:
+ path: test-results

.codecov.yml

@@ -1,6 +1,8 @@
 codecov:
  notify:
  require_ci_to_pass: yes
+ # wait for unit and integration test builds.
+ after_n_builds: 2
  strict_yaml_branch: master # only use the latest copy on master branch
 
 coverage:
@@ -15,4 +17,4 @@ coverage:
  patch:
  default:
  enabled: yes
- target: 95%
+ target: 95%

.gitignore

@@ -25,4 +25,6 @@ bin/
 
 # Coverage
 coverage.txt
+integration-coverage.txt
 coverage.html
+integration-coverage.html

Makefile

@@ -14,6 +14,12 @@ BUILD_INFO=-ldflags "${BUILD_X1} ${BUILD_X2} ${BUILD_X3}"
 STATIC_CHECK=staticcheck
 OTEL_VERSION=master
 
+# Modules to run integration tests on.
+# XXX: Find a way to automatically populate this. Too slow to run across all modules when there are just a few.
+INTEGRATION_TEST_MODULES := \
+ receiver/redisreceiver \
+ internal/common
+
 .DEFAULT_GOAL := all
 
 .PHONY: all
@@ -23,26 +29,20 @@ all: common otelcontribcol
 e2e-test: otelcontribcol
  $(MAKE) -C testbed runtests
 
-.PHONY: precommit
-precommit:
- $(MAKE) gotidy
- $(MAKE) ci
-
 .PHONY: test-with-cover
-test-with-cover:
+unit-tests-with-cover:
  @echo Verifying that all packages have test files to count in coverage
  @scripts/check-test-files.sh $(subst github.com/open-telemetry/opentelemetry-collector-contrib/,./,$(ALL_PKGS))
- @echo pre-compiling tests
- set -e; for dir in $(ALL_MODULES); do \
- echo "go test ./... + coverage in $${dir}"; \
- (cd "$${dir}" && \
- $(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) ./... && \
- go tool cover -html=coverage.txt -o coverage.html ); \
- done
+ @$(MAKE) for-all CMD="make do-unit-tests-with-cover"
+
+.PHONY: integration-tests-with-cover
+integration-tests-with-cover:
+ @echo $(INTEGRATION_TEST_MODULES)
+ @$(MAKE) for-all CMD="make do-integration-tests-with-cover" ALL_MODULES="$(INTEGRATION_TEST_MODULES)"
 
 .PHONY: stability-tests
 stability-tests:
- @echo Stability tests have not been implemented yet 
+ @echo Stability tests have not been implemented yet
 
 .PHONY: gotidy
 gotidy:
@@ -93,9 +93,10 @@ install-tools:
  go install github.com/client9/misspell/cmd/misspell
  go install github.com/golangci/golangci-lint/cmd/golangci-lint
  go install github.com/google/addlicense
- go install honnef.co/go/tools/cmd/staticcheck
+ go install github.com/jstemmer/go-junit-report
  go install github.com/pavius/impi/cmd/impi
  go install github.com/tcnksm/ghr
+ go install honnef.co/go/tools/cmd/staticcheck
 
 .PHONY: run
 run:

Makefile.Common

@@ -15,6 +15,7 @@ ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort
 
 GOTEST_OPT?= -race -timeout 30s
 GOTEST_OPT_WITH_COVERAGE = $(GOTEST_OPT) -coverprofile=coverage.txt -covermode=atomic
+GOTEST_OPT_WITH_INTEGRATION=$(GOTEST_OPT) -v -tags=integration -run=Integration -coverprofile=integration-coverage.txt -covermode=atomic
 GOTEST=go test
 GOOS=$(shell go env GOOS)
 GOARCH=$(shell go env GOARCH)
@@ -46,6 +47,20 @@ test:
  $(GOTEST) ./... ); \
  done
 
+.PHONY: do-unit-tests-with-cover
+do-unit-tests-with-cover:
+ @echo "running go unit test ./... + coverage in `pwd`"
+ @$(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) ./...
+ go tool cover -html=coverage.txt -o coverage.html
+
+.PHONY: run-integration-tests-with-cover
+do-integration-tests-with-cover:
+ @echo "running go integration test ./... + coverage in `pwd`"
+ @$(GOTEST) $(GOTEST_OPT_WITH_INTEGRATION) ./...
+ @if [ -e integration-coverage.txt ]; then \
+ go tool cover -html=integration-coverage.txt -o integration-coverage.html; \
+ fi
+
 .PHONY: benchmark
 benchmark:
  $(GOTEST) -bench=. -run=notests $(ALL_PKGS)

internal/common/integration_test.go

@@ -0,0 +1,25 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build integration
+
+package common
+
+import (
+ "testing"
+)
+
+func TestIntegration(t *testing.T) {
+ t.Log("just for exercising integration coverage, remove in next PR")
+}