[chore] Simplify tool management (open-telemetry#17849)

* Simplifying the tooling practice

* Ensuring that commands that use a global install are moved to use the
generated bin folder

* Using `-p` to gracefully fail

* Moving over to named imports

* Fixing impi

.gitignore

@@ -16,6 +16,7 @@ local/
 *.dylib
 bin/
 dist/
+.tools/
 
 # Emacs
 *~

Makefile

@@ -105,12 +105,12 @@ golint:
  $(MAKE) $(FOR_GROUP_TARGET) TARGET="lint"
 
 .PHONY: goimpi
-goimpi: install-tools
+goimpi: $(IMPI)
  @$(MAKE) $(FOR_GROUP_TARGET) TARGET="impi"
 
 .PHONY: goporto
-goporto: install-tools
- porto -w --include-internal --skip-dirs "^cmd$$" ./
+goporto: $(PORTO)
+ $(PORTO) -w --include-internal --skip-dirs "^cmd$$" ./
 
 .PHONY: for-all
 for-all:
@@ -126,9 +126,9 @@ COMMIT?=HEAD
 MODSET?=contrib-core
 REMOTE?[email protected]:open-telemetry/opentelemetry-collector-contrib.git
 .PHONY: push-tags
-push-tags:
- multimod verify
- set -e; for tag in `multimod tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
+push-tags: $(MULITMOD) 
+ $(MULITMOD) verify
+ set -e; for tag in `$(MULITMOD) tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
  echo "pushing tag $${tag}"; \
  git push ${REMOTE} $${tag}; \
  done;
@@ -203,22 +203,6 @@ for-other-target: $(OTHER_MODS)
 all-pwd:
  $(MAKE) $(FOR_GROUP_TARGET) TARGET="pwd"
 
-TOOLS_MOD_DIR := ./internal/tools
-.PHONY: install-tools
-install-tools:
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/client9/misspell/cmd/misspell
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/google/addlicense
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/jstemmer/go-junit-report
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/pavius/impi/cmd/impi
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/tcnksm/ghr
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/checkdoc
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/issuegenerator
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install golang.org/x/tools/cmd/goimports
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/multimod
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/jcchavezs/porto/cmd/porto
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/crosslink
-
 .PHONY: run
 run:
  cd ./cmd/otelcontribcol && GO111MODULE=on $(GOCMD) run --race . --config ../../${RUN_CONFIG} ${RUN_ARGS}
@@ -250,26 +234,22 @@ mdatagen-test:
  cd cmd/mdatagen && $(GOCMD) install .
  cd cmd/mdatagen && $(GOCMD) generate ./...
 
-.PHONY: chlog-install
-chlog-install:
- cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/chloggen
-
 FILENAME?=$(shell git branch --show-current)
 .PHONY: chlog-new
-chlog-new: chlog-install
- chloggen new --filename $(FILENAME)
+chlog-new: $(CHLOGGEN)
+ $(CHLOGGEN) new --filename $(FILENAME)
 
 .PHONY: chlog-validate
-chlog-validate: chlog-install
- chloggen validate
+chlog-validate: $(CHLOGGEN)
+ $(CHLOGGEN) validate
 
 .PHONY: chlog-preview
-chlog-preview: chlog-install
- chloggen update --dry
+chlog-preview: $(CHLOGGEN)
+ $(CHLOGGEN) update --dry
 
 .PHONY: chlog-update
-chlog-update: chlog-install
- chloggen update --version $(VERSION)
+chlog-update: $(CHLOGGEN)
+ $(CHLOGGEN) update --version $(VERSION)
 
 # Build the Collector executable.
 .PHONY: otelcontribcol
@@ -323,8 +303,8 @@ build-examples:
 
 # Verify existence of READMEs for components specified as default components in the collector.
 .PHONY: checkdoc
-checkdoc:
- checkdoc --project-path $(CURDIR) --component-rel-path $(COMP_REL_PATH) --module-name $(MOD_NAME)
+checkdoc: $(CHECKDOC)
+ $(CHECKDOC) --project-path $(CURDIR) --component-rel-path $(COMP_REL_PATH) --module-name $(MOD_NAME)
 
 .PHONY: all-checklinks
 all-checklinks:
@@ -351,24 +331,24 @@ certs:
  $(foreach dir, $(CERT_DIRS), $(call exec-command, @internal/buildscripts/gen-certs.sh -o $(dir)))
 
 .PHONY: multimod-verify
-multimod-verify: install-tools
+multimod-verify: $(MULITMOD)
  @echo "Validating versions.yaml"
- multimod verify
+ $(MULITMOD) verify
 
 .PHONY: multimod-prerelease
-multimod-prerelease: install-tools
- multimod prerelease -s=true -b=false -v ./versions.yaml -m contrib-base
+multimod-prerelease: $(MULITMOD)
+ $(MULITMOD) prerelease -s=true -b=false -v ./versions.yaml -m contrib-base
  $(MAKE) gotidy
 
 .PHONY: multimod-sync
-multimod-sync: install-tools
- multimod sync -a=true -s=true -o ../opentelemetry-collector
+multimod-sync: $(MULITMOD)
+ $(MULITMOD) sync -a=true -s=true -o ../opentelemetry-collector
  $(MAKE) gotidy
 
 .PHONY: crosslink
-crosslink: install-tools
+crosslink: $(CROSSLINK)
  @echo "Executing crosslink"
- crosslink --root=$(shell pwd)
+ $(CROSSLINK) --root=$(shell pwd)
 
 .PHONY: clean
 clean:

Makefile.Common

@@ -22,12 +22,40 @@ GOCMD?= go
 GOTEST=$(GOCMD) test
 GOOS=$(shell $(GOCMD) env GOOS)
 GOARCH=$(shell $(GOCMD) env GOARCH)
-ADDLICENCESE= addlicense
-MDLINKCHECK=markdown-link-check
-MISSPELL=misspell -error
-MISSPELL_CORRECTION=misspell -w
-LINT=golangci-lint
-IMPI=impi
+
+# In order to help reduce toil related to managing tooling for the open telemetry collector
+# this section of the makefile looks at only requiring command definitions to be defined
+# as part of $(TOOLS_MOD_DIR)/tools.go, following the existing practice.
+# Modifying the tools' `go.mod` file will trigger a rebuild of the tools to help
+# ensure that all contributors are using the most recent version to make builds repeatable everywhere.
+TOOLS_MOD_DIR := $(PWD)/internal/tools
+TOOLS_MOD_REGEX := "\s+_\s+\".*\""
+TOOLS_PKG_NAMES := $(shell grep -E $(TOOLS_MOD_REGEX) < $(TOOLS_MOD_DIR)/tools.go | tr -d " _\"")
+TOOLS_BIN_DIR := $(PWD)/.tools
+TOOLS_BIN_NAMES := $(addprefix $(TOOLS_BIN_DIR)/, $(notdir $(TOOLS_PKG_NAMES)))
+
+.PHONY: install-tools
+install-tools: $(TOOLS_BIN_NAMES)
+
+$(TOOLS_BIN_DIR):
+ mkdir -p $(PWD)/bin
+
+$(TOOLS_BIN_NAMES): $(TOOLS_BIN_DIR) $(TOOLS_MOD_DIR)/go.mod
+ cd $(TOOLS_MOD_DIR) && $(GOCMD) build -o $@ -trimpath $(filter %/$(notdir $@),$(TOOLS_PKG_NAMES))
+
+ADDLICENCESE := $(TOOLS_BIN_DIR)/addlicense
+MDLINKCHECK := $(TOOLS_BIN_DIR)/markdown-link-check
+MISSPELL := $(TOOLS_BIN_DIR)/misspell -error
+MISSPELL_CORRECTION := $(TOOLS_BIN_DIR)/misspell -w
+LINT := $(TOOLS_BIN_DIR)/golangci-lint
+IMPI := $(TOOLS_BIN_DIR)/impi
+MULITMOD := $(TOOLS_BIN_DIR)/multimod
+CHLOGGEN := $(TOOLS_BIN_DIR)/chloggen
+GOIMPORTS := $(TOOLS_BIN_DIR)/goimports
+PORTO := $(TOOLS_BIN_DIR)/porto
+CHECKDOC := $(TOOLS_BIN_DIR)/checkdoc
+CROSSLINK := $(TOOLS_BIN_DIR)/crosslink
+
 # BUILD_TYPE should be one of (dev, release).
 BUILD_TYPE?=release
 RUNNING_ON_GITHUB_ACTION=$(GITHUB_ACTIONS)
@@ -99,7 +127,7 @@ benchmark:
  $(GOTEST) -bench=. -run=notests --tags=$(GO_BUILD_TAGS) $(ALL_PKGS)
 
 .PHONY: addlicense
-addlicense:
+addlicense: $(ADDLICENCESE)
  @ADDLICENCESEOUT=`$(ADDLICENCESE) -y "" -c 'The OpenTelemetry Authors' $(ALL_SRC) 2>&1`; \
  if [ "$$ADDLICENCESEOUT" ]; then \
  echo "$(ADDLICENCESE) FAILED => add License errors:\n"; \
@@ -110,7 +138,7 @@ addlicense:
  fi
 
 .PHONY: checklicense
-checklicense:
+checklicense: $(ADDLICENCESE)
  @ADDLICENCESEOUT=`$(ADDLICENCESE) -check $(ALL_SRC) 2>&1`; \
  if [ "$$ADDLICENCESEOUT" ]; then \
  echo "$(ADDLICENCESE) FAILED => add License errors:\n"; \
@@ -128,12 +156,12 @@ checklinks:
  $(MDLINKCHECK) -q -c $(SRC_ROOT)/.github/workflows/check_links_config.json || true
 
 .PHONY: fmt
-fmt:
+fmt: $(GOIMPORTS)
  gofmt -w -s ./
- goimports -w -local github.com/open-telemetry/opentelemetry-collector-contrib ./
+ $(GOIMPORTS) -w -local github.com/open-telemetry/opentelemetry-collector-contrib ./
 
 .PHONY: lint
-lint: checklicense misspell
+lint: $(LINT) checklicense misspell
  $(LINT) run --allow-parallel-runners
 
 .PHONY: tidy
@@ -142,16 +170,16 @@ tidy:
  $(GOCMD) mod tidy -compat=1.18
 
 .PHONY: misspell
-misspell:
+misspell: $(TOOLS_BIN_DIR)/misspell
  @echo "running $(MISSPELL)"
  @$(MISSPELL) $(ALL_SRC_AND_DOC)
 
 .PHONY: misspell-correction
-misspell-correction:
+misspell-correction: $(TOOLS_BIN_DIR)/misspell
  $(MISSPELL_CORRECTION) $(ALL_SRC_AND_DOC)
 
 .PHONY: impi
-impi:
+impi: $(IMPI)
  @$(IMPI) --local github.com/open-telemetry/opentelemetry-collector-contrib --scheme stdThirdPartyLocal ./...
 
 .PHONY: moddownload