cmd/go: add support for vendoring in workspace mode

In most cases this change removes assumptions that there is a single
main module in vendor mode and iterates over the workspace modules
when doing checks. The go mod vendor command will now, if in workspace
mode, create a vendor directory in the same directory as the go.work
file, containing the packages (and modules in modules.txt) loaded from
the workspace. When reassembling the module graph from the vendor
directory, an edges are added from each of the main modules to their
requirements, plus additionally to a fake 'vendor/modules.txt' module
with edges to all the modules listed in vendor/modules.txt.

For #60056

Change-Id: I4a485bb39836e7ab35cdc7726229191c6599903e
Reviewed-on: https://go-review.googlesource.com/c/go/+/495801
Reviewed-by: Bryan Mills <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Michael Matloob <[email protected]>

src/cmd/go/internal/modcmd/vendor.go

@@ -66,6 +66,14 @@ func init() {
 }
 
 func runVendor(ctx context.Context, cmd *base.Command, args []string) {
+ modload.InitWorkfile()
+ if modload.WorkFilePath() != "" {
+ base.Fatalf("go: 'go mod vendor' cannot be run in workspace mode. Run 'go work vendor' to vendor the workspace or set 'GOWORK=off' to exit workspace mode.")
+ }
+ RunVendor(ctx, vendorE, vendorO, args)
+}
+
+func RunVendor(ctx context.Context, vendorE bool, vendorO string, args []string) {
  if len(args) != 0 {
  base.Fatalf("go: 'go mod vendor' accepts no arguments")
  }
@@ -98,7 +106,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
  modpkgs := make(map[module.Version][]string)
  for _, pkg := range pkgs {
  m := modload.PackageModule(pkg)
- if m.Path == "" || m.Version == "" && modload.MainModules.Contains(m.Path) {
+ if m.Path == "" || modload.MainModules.Contains(m.Path) {
  continue
  }
  modpkgs[m] = append(modpkgs[m], pkg)
@@ -107,21 +115,25 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
  includeAllReplacements := false
  includeGoVersions := false
  isExplicit := map[module.Version]bool{}
- if gv := modload.ModFile().Go; gv != nil {
- if gover.Compare(gv.Version, "1.14") >= 0 {
- // If the Go version is at least 1.14, annotate all explicit 'require' and
- // 'replace' targets found in the go.mod file so that we can perform a
- // stronger consistency check when -mod=vendor is set.
- for _, r := range modload.ModFile().Require {
- isExplicit[r.Mod] = true
+ gv := modload.MainModules.GoVersion()
+ if gover.Compare(gv, "1.14") >= 0 && (modload.FindGoWork(base.Cwd()) != "" || modload.ModFile().Go != nil) {
+ // If the Go version is at least 1.14, annotate all explicit 'require' and
+ // 'replace' targets found in the go.mod file so that we can perform a
+ // stronger consistency check when -mod=vendor is set.
+ for _, m := range modload.MainModules.Versions() {
+ if modFile := modload.MainModules.ModFile(m); modFile != nil {
+ for _, r := range modFile.Require {
+ isExplicit[r.Mod] = true
+ }
  }
- includeAllReplacements = true
- }
- if gover.Compare(gv.Version, "1.17") >= 0 {
- // If the Go version is at least 1.17, annotate all modules with their
- // 'go' version directives.
- includeGoVersions = true
+
  }
+ includeAllReplacements = true
+ }
+ if gover.Compare(gv, "1.17") >= 0 {
+ // If the Go version is at least 1.17, annotate all modules with their
+ // 'go' version directives.
+ includeGoVersions = true
  }
 
  var vendorMods []module.Version
@@ -143,9 +155,11 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
  w = io.MultiWriter(&buf, os.Stderr)
  }
 
+ replacementWritten := make(map[module.Version]bool)
  for _, m := range vendorMods {
  replacement := modload.Replacement(m)
  line := moduleLine(m, replacement)
+ replacementWritten[m] = true
  io.WriteString(w, line)
 
  goVersion := ""
@@ -173,17 +187,41 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
  // Record unused and wildcard replacements at the end of the modules.txt file:
  // without access to the complete build list, the consumer of the vendor
  // directory can't otherwise determine that those replacements had no effect.
- for _, r := range modload.ModFile().Replace {
- if len(modpkgs[r.Old]) > 0 {
- // We we already recorded this replacement in the entry for the replaced
- // module with the packages it provides.
- continue
+ for _, m := range modload.MainModules.Versions() {
+ if workFile := modload.MainModules.WorkFile(); workFile != nil {
+ for _, r := range workFile.Replace {
+ if replacementWritten[r.Old] {
+ // We already recorded this replacement.
+ continue
+ }
+ replacementWritten[r.Old] = true
+
+ line := moduleLine(r.Old, r.New)
+ buf.WriteString(line)
+ if cfg.BuildV {
+ os.Stderr.WriteString(line)
+ }
+ }
  }
-
- line := moduleLine(r.Old, r.New)
- buf.WriteString(line)
- if cfg.BuildV {
- os.Stderr.WriteString(line)
+ if modFile := modload.MainModules.ModFile(m); modFile != nil {
+ for _, r := range modFile.Replace {
+ if replacementWritten[r.Old] {
+ // We already recorded this replacement.
+ continue
+ }
+ replacementWritten[r.Old] = true
+ rNew := modload.Replacement(r.Old)
+ if rNew == (module.Version{}) {
+ // There is no replacement. Don't try to write it.
+ continue
+ }
+
+ line := moduleLine(r.Old, rNew)
+ buf.WriteString(line)
+ if cfg.BuildV {
+ os.Stderr.WriteString(line)
+ }
+ }
  }
  }
  }
@@ -367,7 +405,7 @@ func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
  return false
  }
  if info.Name() == "go.mod" || info.Name() == "go.sum" {
- if gv := modload.ModFile().Go; gv != nil && gover.Compare(gv.Version, "1.17") >= 0 {
+ if gv := modload.MainModules.GoVersion(); gover.Compare(gv, "1.17") >= 0 {
  // As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
  // Otherwise, 'go' commands invoked within the vendor subtree may misidentify
  // an arbitrary directory within the vendor tree as a module root.

src/cmd/go/internal/modload/buildlist.go

@@ -174,17 +174,20 @@ func (rs *Requirements) String() string {
 // requirements.
 func (rs *Requirements) initVendor(vendorList []module.Version) {
  rs.graphOnce.Do(func() {
- mg := &ModuleGraph{
- g: mvs.NewGraph(cmpVersion, MainModules.Versions()),
+ roots := MainModules.Versions()
+ if inWorkspaceMode() {
+ // Use rs.rootModules to pull in the go and toolchain roots
+ // from the go.work file and preserve the invariant that all
+ // of rs.rootModules are in mg.g.
+ roots = rs.rootModules
  }
-
- if MainModules.Len() != 1 {
- panic("There should be exactly one main module in Vendor mode.")
+ mg := &ModuleGraph{
+ g: mvs.NewGraph(cmpVersion, roots),
  }
- mainModule := MainModules.Versions()[0]
 
  if rs.pruning == pruned {
- // The roots of a pruned module should already include every module in the
+ mainModule := MainModules.mustGetSingleMainModule()
+ // The roots of a single pruned module should already include every module in the
  // vendor list, because the vendored modules are the same as those needed
  // for graph pruning.
  //
@@ -215,8 +218,18 @@ func (rs *Requirements) initVendor(vendorList []module.Version) {
  // graph, but still distinguishes between direct and indirect
  // dependencies.
  vendorMod := module.Version{Path: "vendor/modules.txt", Version: ""}
- mg.g.Require(mainModule, append(rs.rootModules, vendorMod))
- mg.g.Require(vendorMod, vendorList)
+ if inWorkspaceMode() {
+ for _, m := range MainModules.Versions() {
+ reqs, _ := rootsFromModFile(m, MainModules.ModFile(m), omitToolchainRoot)
+ mg.g.Require(m, append(reqs, vendorMod))
+ }
+ mg.g.Require(vendorMod, vendorList)
+
+ } else {
+ mainModule := MainModules.mustGetSingleMainModule()
+ mg.g.Require(mainModule, append(rs.rootModules, vendorMod))
+ mg.g.Require(vendorMod, vendorList)
+ }
  }
 
  rs.graph.Store(&cachedGraph{mg, nil})

src/cmd/go/internal/modload/import.go

@@ -318,38 +318,49 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M
  mods = append(mods, module.Version{})
  }
  // -mod=vendor is special.
- // Everything must be in the main module or the main module's vendor directory.
+ // Everything must be in the main modules or the main module's or workspace's vendor directory.
  if cfg.BuildMod == "vendor" {
- mainModule := MainModules.mustGetSingleMainModule()
- modRoot := MainModules.ModRoot(mainModule)
  var mainErr error
- if modRoot != "" {
- mainDir, mainOK, err := dirInModule(path, MainModules.PathPrefix(mainModule), modRoot, true)
- mainErr = err
- if mainOK {
- mods = append(mods, mainModule)
- dirs = append(dirs, mainDir)
- roots = append(roots, modRoot)
+ for _, mainModule := range MainModules.Versions() {
+ modRoot := MainModules.ModRoot(mainModule)
+ if modRoot != "" {
+ dir, mainOK, err := dirInModule(path, MainModules.PathPrefix(mainModule), modRoot, true)
+ if mainErr == nil {
+ mainErr = err
+ }
+ if mainOK {
+ mods = append(mods, mainModule)
+ dirs = append(dirs, dir)
+ roots = append(roots, modRoot)
+ }
  }
- vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(modRoot, "vendor"), false)
+ }
+
+ if HasModRoot() {
+ vendorDir := VendorDir()
+ dir, vendorOK, _ := dirInModule(path, "", vendorDir, false)
  if vendorOK {
- readVendorList(mainModule)
+ readVendorList(vendorDir)
+ // TODO(#60922): It's possible for a package to manually have been added to the
+ // vendor directory, causing the dirInModule to succeed, but no vendorPkgModule
+ // to exist, causing an empty module path to be reported. Do better checking
+ // here.
  mods = append(mods, vendorPkgModule[path])
- dirs = append(dirs, vendorDir)
- roots = append(roots, modRoot)
+ dirs = append(dirs, dir)
+ roots = append(roots, vendorDir)
  }
  }
 
  if len(dirs) > 1 {
- return module.Version{}, modRoot, "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs}
+ return module.Version{}, "", "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs}
  }
 
  if mainErr != nil {
  return module.Version{}, "", "", nil, mainErr
  }
 
  if len(dirs) == 0 {
- return module.Version{}, modRoot, "", nil, &ImportMissingError{Path: path}
+ return module.Version{}, "", "", nil, &ImportMissingError{Path: path}
  }
 
  return mods[0], roots[0], dirs[0], nil, nil