Remove vendor (sourcegraph#379)

* git rm -r vendor

* ci: Remove go mod vendor check

* all: remove direct uses of vendor

* all: Specify GO111MODULE=on and GOBIN for generate

* ci: Enable go modules when generating pipeline

* ci: Enable go modules for every pipeline step

* ci: Use count flag to force test run instead of disabling GOCACHE

* ci: Use next branch of honnef.co/go/tools

master currently does not support go modules, but next should.

* all: Update to pass next branch of staticcheck

cmd/frontend/db/gen.go

@@ -1,4 +1,4 @@
 package db
 
 // $PGHOST, $PGUSER, $PGPORT etc. must be set to run this generate script.
-//go:generate go run schemadoc/main.go schema.md
+//go:generate env GO111MODULE=on go run schemadoc/main.go schema.md

cmd/frontend/db/migrations/gen.go

@@ -1,5 +1,5 @@
 package migrations
 
-//go:generate go build -o ../../../../vendor/.bin/go-bindata github.com/sourcegraph/sourcegraph/vendor/github.com/kevinburke/go-bindata/go-bindata
-//go:generate ../../../../vendor/.bin/go-bindata -nometadata -pkg migrations -prefix ../../../../migrations/ -ignore README.md ../../../../migrations/
+//go:generate env GOBIN=$PWD/.bin GO111MODULE=on go install github.com/kevinburke/go-bindata/go-bindata
+//go:generate $PWD/.bin/go-bindata -nometadata -pkg migrations -prefix ../../../../migrations/ -ignore README.md ../../../../migrations/
 //go:generate gofmt -w bindata.go

cmd/frontend/graphqlbackend/gen.go

@@ -1,3 +1,3 @@
 package graphqlbackend
 
-//go:generate go run schema_generate.go
+//go:generate env GO111MODULE=on go run schema_generate.go

cmd/frontend/internal/app/assets/gen.go

@@ -1,3 +1,3 @@
-//go:generate go run assets_generate.go
+//go:generate env GO111MODULE=on go run assets_generate.go
 
 package assets

cmd/frontend/internal/app/templates/gen.go

@@ -1,3 +1,3 @@
-//go:generate go run data_generate.go
+//go:generate env GO111MODULE=on go run data_generate.go
 
 package templates

cmd/frontend/internal/pkg/search/query/syntax/scanner.go

@@ -9,7 +9,8 @@ import (
 // TokenType is the set of lexical tokens in the query syntax.
 type TokenType int
 
-//go:generate stringer -type=TokenType
+//go:generate env GOBIN=$PWD/.bin GO111MODULE=on go install golang.org/x/tools/cmd/stringer
+//go:generate env $PWD/.bin/stringer -type=TokenType
 
 // All TokenType values.
 const (

cmd/server/build.sh

@@ -4,7 +4,8 @@
 cd $(dirname "${BASH_SOURCE[0]}")/../..
 set -ex
 
-GOBIN=$PWD/vendor/.bin go install ./vendor/github.com/sourcegraph/godockerize
+export GO111MODULE=on
+GOBIN=$PWD/.bin go install github.com/sourcegraph/godockerize
 
 # Additional images passed in here when this script is called externally by our
 # enterprise build scripts.
@@ -14,7 +15,7 @@ additional_images=${@:-github.com/sourcegraph/sourcegraph/cmd/frontend}
 # our enterprise build scripts.
 server_pkg=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/cmd/server}
 
-./vendor/.bin/godockerize build --base 'alpine:3.8' -t ${IMAGE} --go-build-flags="-ldflags" --go-build-flags="-X github.com/sourcegraph/sourcegraph/pkg/version.version=${VERSION}" --env VERSION=${VERSION} \
+./.bin/godockerize build --base 'alpine:3.8' -t ${IMAGE} --go-build-flags="-ldflags" --go-build-flags="-X github.com/sourcegraph/sourcegraph/pkg/version.version=${VERSION}" --env VERSION=${VERSION} --env GO111MODULES=on \
  $server_pkg \
  github.com/sourcegraph/sourcegraph/cmd/github-proxy \
  github.com/sourcegraph/sourcegraph/cmd/gitserver \
@@ -23,7 +24,7 @@ server_pkg=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/cmd/server}
  github.com/sourcegraph/sourcegraph/cmd/repo-updater \
  github.com/sourcegraph/sourcegraph/cmd/searcher \
  github.com/sourcegraph/sourcegraph/cmd/indexer \
- github.com/sourcegraph/sourcegraph/vendor/github.com/google/zoekt/cmd/zoekt-archive-index \
- github.com/sourcegraph/sourcegraph/vendor/github.com/google/zoekt/cmd/zoekt-sourcegraph-indexserver \
- github.com/sourcegraph/sourcegraph/vendor/github.com/google/zoekt/cmd/zoekt-webserver \
+ github.com/google/zoekt/cmd/zoekt-archive-index \
+ github.com/google/zoekt/cmd/zoekt-sourcegraph-indexserver \
+ github.com/google/zoekt/cmd/zoekt-webserver \
  github.com/sourcegraph/sourcegraph/cmd/lsp-proxy $additional_images

dev/check/all.sh

@@ -9,7 +9,6 @@ go env
 ./template-inlines.sh
 ./go-generate.sh
 ./go-lint.sh
-./go-mod.sh
 ./todo-security.sh
 ./no-localhost-guard.sh
 ./bash-syntax.sh

dev/check/go-generate.sh

@@ -5,11 +5,9 @@ set -eo pipefail
 main() {
  cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 
- export GOBIN="$PWD/vendor/.bin"
+ export GOBIN="$PWD/.bin"
  export PATH=$GOBIN:$PATH
-
- go install github.com/sourcegraph/sourcegraph/vendor/golang.org/x/tools/cmd/stringer
- go install github.com/sourcegraph/sourcegraph/vendor/github.com/sourcegraph/go-jsonschema/cmd/go-jsonschema-compiler
+ export GO111MODULE=on
 
  # Runs generate.sh and ensures no files changed. This relies on the go
  # generation that ran are idempotent.

dev/check/go-lint.sh

@@ -2,16 +2,17 @@
 set -e
 cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 
-export GOBIN="$PWD/vendor/.bin"
+export GOBIN="$PWD/.bin"
 export PATH=$GOBIN:$PATH
+export GO111MODULE=on
 
-go install github.com/sourcegraph/sourcegraph/vendor/honnef.co/go/tools/cmd/megacheck
+go install honnef.co/go/tools/cmd/staticcheck
 
 echo go install...
 go install -buildmode=archive ./...
 
 echo go vet...
 go vet ./...
 
-echo staticcheck and unused...
-megacheck --simple.exit-non-zero=true ./...
+echo staticcheck...
+staticcheck -ignore '*:ST*' ./...

dev/ci/ci-db-backcompat.sh

@@ -34,7 +34,7 @@ echo ""
 # Recreate the test DB and run TestMigrations once to ensure that the schema version is the latest.
 set -ex
 ./dev/ci/reset-test-db.sh || true
-GOCACHE=off go test -v ./cmd/frontend/db/ -run=TestMigrations
+go test -count=1 -v ./cmd/frontend/db/ -run=TestMigrations
 HEAD="$HEAD" OLD="${COMMIT_BEFORE_LAST_MIGRATION}" ./dev/ci/db-backcompat.sh
 set +ex
 

dev/ci/db-backcompat.sh

@@ -42,7 +42,7 @@ CURRENT_DB_SCHEMA=$(psql -t -d sourcegraph-test-db -c 'select version from schem
 
 if [ "$LATEST_SCHEMA" != "$CURRENT_DB_SCHEMA" ]; then
  echo "Latest migration schema version ($LATEST_SCHEMA) does not match schema in test DB ($CURRENT_DB_SCHEMA)."
- echo ' You can run `GOCACHE=off go test -v ./cmd/frontend/db/ -run=TestMigrations` to update the test DB schema.'
+ echo ' You can run `go test -count=1 -v ./cmd/frontend/db/ -run=TestMigrations` to update the test DB schema.'
  exit 1
 fi
 
@@ -63,7 +63,7 @@ function runTest() {
  set -ex
  # Test without cache, because schema change does not
  # necessarily mean Go source has changed.
- GOCACHE=off SKIP_MIGRATION_TEST=true go test -v ./cmd/frontend/db/
+ SKIP_MIGRATION_TEST=true go test -count=1 -v ./cmd/frontend/db/
  set +ex
 
  NOW_DB_SCHEMA=$(psql -t -d sourcegraph-test-db -c 'select version from schema_migrations' | xargs echo -n)

dev/ci/gen-pipeline.go

@@ -38,6 +38,9 @@ func main() {
 
  branch := os.Getenv("BUILDKITE_BRANCH")
 
+ bk.OnEveryStepOpts = append(bk.OnEveryStepOpts,
+ bk.Env("GO111MODULE", "on"))
+
  pipeline.AddStep(":white_check_mark:",
  bk.Cmd("./dev/check/all.sh"))
 
@@ -79,7 +82,7 @@ func main() {
 
  pipeline.AddStep(":docker:",
  bk.Cmd("curl -sL -o hadolint \"https://github.com/hadolint/hadolint/releases/download/v1.6.5/hadolint-$(uname -s)-$(uname -m)\" && chmod 700 hadolint"),
- bk.Cmd("git ls-files | grep -v '^vendor/' | grep Dockerfile | xargs ./hadolint"))
+ bk.Cmd("git ls-files | grep Dockerfile | xargs ./hadolint"))
 
  pipeline.AddStep(":postgres:",
  bk.Cmd("./dev/ci/ci-db-backcompat.sh"))

dev/ci/init-pipeline.yml

@@ -1,6 +1,6 @@
 steps:
  - command: |
- go run ./dev/ci/gen-pipeline.go | tee pipeline.yml
+ GO111MODULE=on go run ./dev/ci/gen-pipeline.go | tee pipeline.yml
  buildkite-agent pipeline upload pipeline.yml
  label: ':pipeline:'
  plugins:

dev/generate.sh

@@ -2,4 +2,4 @@
 
 cd "$(dirname "${BASH_SOURCE[0]}")/.." # cd to repo root dir
 
-go list ./... | grep -v /vendor/ | xargs go generate -v
+go list ./... | xargs go generate

dev/tools.go

@@ -12,7 +12,5 @@ import (
  _ "github.com/sourcegraph/go-jsonschema/cmd/go-jsonschema-compiler"
  _ "github.com/sourcegraph/godockerize"
  _ "golang.org/x/tools/cmd/stringer"
- _ "honnef.co/go/tools/cmd/megacheck"
  _ "honnef.co/go/tools/cmd/staticcheck"
- _ "honnef.co/go/tools/cmd/unused"
 )

docs/local-development.md

@@ -349,10 +349,7 @@ Known issues:
 
 ## Go dependency management
 
-We use Go modules to manage Go dependencies in this repository. The CI test
-suite will check whether you have updated `go.mod` and `go.sum` correctly - in
-particular, running `go mod tidy && go mod vendor` should not generate a Git
-diff.
+We use Go modules to manage Go dependencies in this repository.
 
 ## Codegen
 

go.mod

@@ -1,6 +1,7 @@
 module github.com/sourcegraph/sourcegraph
 
 require (
+ github.com/BurntSushi/toml v0.3.1 // indirect
  github.com/NYTimes/gziphandler v1.0.1
  github.com/aws/aws-sdk-go-v2 v2.0.0-preview.4+incompatible
  github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
@@ -121,7 +122,7 @@ require (
  gopkg.in/src-d/go-git.v4 v4.6.0
  gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 // indirect
  gopkg.in/yaml.v2 v2.2.1
- honnef.co/go/tools v0.0.0-20180728063816-88497007e858
+ honnef.co/go/tools v0.0.0-20180910201051-f1b53a58b022
  sourcegraph.com/sourcegraph/go-diff v0.0.0-20171119081133-3f415a150aec
  sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 // indirect
 )

go.sum

@@ -1,4 +1,6 @@
 cloud.google.com/go v0.23.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.4.9 h1:3RbgqgGVqmcpbOiwrjbVtDHLlJBGF6aE+yHmNtBNsFQ=
 github.com/Microsoft/go-winio v0.4.9/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/NYTimes/gziphandler v1.0.1 h1:iLrQrdwjDd52kHDA5op2UBJFjmOb9g+7scBan4RN8F0=
@@ -395,6 +397,8 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858 h1:wN+eVZ7U+gqdqkec6C6VXR1OFf9a5Ul9ETzeYsYv20g=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20180910201051-f1b53a58b022 h1:6BZ1uHan+VqtcF3xb+hOvKSWhvQy/nk+ZRtQJJLBWIM=
+honnef.co/go/tools v0.0.0-20180910201051-f1b53a58b022/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 sourcegraph.com/sourcegraph/go-diff v0.0.0-20171119081133-3f415a150aec h1:wAAdENPXC7bE1oxY4VqSDdhaA+XQ8TgQHsZMMnrXjEk=
 sourcegraph.com/sourcegraph/go-diff v0.0.0-20171119081133-3f415a150aec/go.mod h1:R09mWeb9JcPbO+A3cYDc11xjz0wp6r9+KnqdqROAoRU=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c=

pkg/inventory/filelang/gen/generate.go

@@ -21,7 +21,7 @@ var (
  in = flag.String("in", "_data", "input directory (with languages.yml, etc.)")
  out = flag.String("out", "data_%s.go", "output filename pattern")
 
- header = fmt.Sprintf("// Code generated by \"go run generate.go %s\"; DO NOT EDIT\n\n//lint:file-ignore SA6004 generated\n", strings.Join(os.Args[1:], " "))
+ header = fmt.Sprintf("// Code generated by \"go run generate.go %s\"; DO NOT EDIT\n\n//lint:file-ignore SA6004 generated\n//lint:file-ignore S1007 generated\n", strings.Join(os.Args[1:], " "))
 )
 
 func main() {

pkg/inventory/filelang/languages.go

@@ -11,7 +11,7 @@ import (
  "gopkg.in/yaml.v2"
 )
 
-//go:generate go run gen/generate.go
+//go:generate env GO111MODULE=on go run gen/generate.go
 
 // A Language represents a programming or markup language.
 //

schema/gen.go

@@ -1,9 +1,9 @@
 package schema
 
-//go:generate go build -o .bin/go-jsonschema-compiler github.com/sourcegraph/sourcegraph/vendor/github.com/sourcegraph/go-jsonschema/cmd/go-jsonschema-compiler
-//go:generate .bin/go-jsonschema-compiler -o schema.go -pkg schema site.schema.json settings.schema.json extension.schema.json
+//go:generate env GOBIN=$PWD/.bin GO111MODULE=on go install github.com/sourcegraph/go-jsonschema/cmd/go-jsonschema-compiler
+//go:generate $PWD/.bin/go-jsonschema-compiler -o schema.go -pkg schema site.schema.json settings.schema.json extension.schema.json
 
-//go:generate go run stringdata.go -i site.schema.json -name SiteSchemaJSON -pkg schema -o site_stringdata.go
-//go:generate go run stringdata.go -i settings.schema.json -name SettingsSchemaJSON -pkg schema -o settings_stringdata.go
-//go:generate go run stringdata.go -i extension.schema.json -name ExtensionSchemaJSON -pkg schema -o extension_stringdata.go
+//go:generate env GO111MODULE=on go run stringdata.go -i site.schema.json -name SiteSchemaJSON -pkg schema -o site_stringdata.go
+//go:generate env GO111MODULE=on go run stringdata.go -i settings.schema.json -name SettingsSchemaJSON -pkg schema -o settings_stringdata.go
+//go:generate env GO111MODULE=on go run stringdata.go -i extension.schema.json -name ExtensionSchemaJSON -pkg schema -o extension_stringdata.go
 //go:generate gofmt -w site_stringdata.go settings_stringdata.go extension_stringdata.go