Merge remote-tracking branch 'origin/master' into debug-exploratory-PR

doc/manual/src/release-notes/rl-next.md

@@ -14,3 +14,13 @@
 
 * `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
  This matches the default behaviour of `nix-build`.
+
+* You can now specify which outputs of a derivation `nix` should
+ operate on using the syntax `installable^outputs`,
+ e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
+ `nix` will use the outputs specified by the derivation's
+ `meta.outputsToInstall` attribute if it exists, or all outputs
+ otherwise.
+
+ Selecting derivation outputs using the attribute selection syntax
+ (e.g. `nixpkgs#glibc.dev`) no longer works.

src/libcmd/installables.cc

@@ -440,10 +440,8 @@ DerivedPaths InstallableValue::toDerivedPaths()
 
  // Group by derivation, helps with .all in particular
  for (auto & drv : toDerivations()) {
- auto outputName = drv.outputName;
- if (outputName == "")
- throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
- drvsToOutputs[drv.drvPath].insert(outputName);
+ for (auto & outputName : drv.outputsToInstall)
+ drvsToOutputs[drv.drvPath].insert(outputName);
  drvsToCopy.insert(drv.drvPath);
  }
 
@@ -466,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
  SourceExprCommand & cmd;
  RootValue v;
  std::string attrPath;
-
- InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
- : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
+ OutputsSpec outputsSpec;
+
+ InstallableAttrPath(
+ ref<EvalState> state,
+ SourceExprCommand & cmd,
+ Value * v,
+ const std::string & attrPath,
+ OutputsSpec outputsSpec)
+ : InstallableValue(state)
+ , cmd(cmd)
+ , v(allocRootValue(v))
+ , attrPath(attrPath)
+ , outputsSpec(std::move(outputsSpec))
  { }
 
  std::string what() const override { return attrPath; }
@@ -497,7 +505,19 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
  auto drvPath = drvInfo.queryDrvPath();
  if (!drvPath)
  throw Error("'%s' is not a derivation", what());
- res.push_back({ *drvPath, drvInfo.queryOutputName() });
+
+ std::set<std::string> outputsToInstall;
+
+ if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+ outputsToInstall = *outputNames;
+ else
+ for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
+ outputsToInstall.insert(output.first);
+
+ res.push_back(DerivationInfo {
+ .drvPath = *drvPath,
+ .outputsToInstall = std::move(outputsToInstall)
+ });
  }
 
  return res;
@@ -574,13 +594,15 @@ InstallableFlake::InstallableFlake(
  ref<EvalState> state,
  FlakeRef && flakeRef,
  std::string_view fragment,
+ OutputsSpec outputsSpec,
  Strings attrPaths,
  Strings prefixes,
  const flake::LockFlags & lockFlags)
  : InstallableValue(state),
  flakeRef(flakeRef),
  attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
  prefixes(fragment == "" ? Strings{} : prefixes),
+ outputsSpec(std::move(outputsSpec)),
  lockFlags(lockFlags)
 {
  if (cmd && cmd->getAutoArgs(*state)->size())
@@ -598,9 +620,29 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
 
  auto drvPath = attr->forceDerivation();
 
+ std::set<std::string> outputsToInstall;
+
+ if (auto aMeta = attr->maybeGetAttr(state->sMeta))
+ if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
+ for (auto & s : aOutputsToInstall->getListOfStrings())
+ outputsToInstall.insert(s);
+
+ if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
+ outputsToInstall.clear();
+ if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
+ for (auto & s : aOutputs->getListOfStrings())
+ outputsToInstall.insert(s);
+ }
+
+ if (outputsToInstall.empty())
+ outputsToInstall.insert("out");
+
+ if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+ outputsToInstall = *outputNames;
+
  auto drvInfo = DerivationInfo {
- std::move(drvPath),
- attr->getAttr(state->sOutputName)->getString()
+ .drvPath = std::move(drvPath),
+ .outputsToInstall = std::move(outputsToInstall),
  };
 
  return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};
@@ -723,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
  state->eval(e, *vFile);
  }
 
- for (auto & s : ss)
- result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
+ for (auto & s : ss) {
+ auto [prefix, outputsSpec] = parseOutputsSpec(s);
+ result.push_back(
+ std::make_shared<InstallableAttrPath>(
+ state, *this, vFile,
+ prefix == "." ? "" : prefix,
+ outputsSpec));
+ }
 
  } else {
 
@@ -743,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
  }
 
  try {
- auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
+ auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
  result.push_back(std::make_shared<InstallableFlake>(
  this,
  getEvalState(),
  std::move(flakeRef),
  fragment,
+ outputsSpec,
  getDefaultFlakeAttrPaths(),
  getDefaultFlakeAttrPathPrefixes(),
  lockFlags));
@@ -822,12 +871,13 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
  auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
  auto drvOutputs = drv.outputsAndOptPaths(*store);
  for (auto & output : bfd.outputs) {
- if (!outputHashes.count(output))
+ auto outputHash = get(outputHashes, output);
+ if (!outputHash)
  throw Error(
  "the derivation '%s' doesn't have an output named '%s'",
  store->printStorePath(bfd.drvPath), output);
  if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
- DrvOutput outputId { outputHashes.at(output), output };
+ DrvOutput outputId { *outputHash, output };
  auto realisation = store->queryRealisation(outputId);
  if (!realisation)
  throw Error(
@@ -838,10 +888,11 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
  } else {
  // If ca-derivations isn't enabled, assume that
  // the output path is statically known.
- assert(drvOutputs.count(output));
- assert(drvOutputs.at(output).second);
+ auto drvOutput = get(drvOutputs, output);
+ assert(drvOutput);
+ assert(drvOutput->second);
  outputs.insert_or_assign(
- output, *drvOutputs.at(output).second);
+ output, *drvOutput->second);
  }
  }
  res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});

src/libcmd/installables.hh

@@ -141,7 +141,7 @@ struct InstallableValue : Installable
  struct DerivationInfo
  {
  StorePath drvPath;
- std::string outputName;
+ std::set<std::string> outputsToInstall;
  };
 
  virtual std::vector<DerivationInfo> toDerivations() = 0;
@@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
  FlakeRef flakeRef;
  Strings attrPaths;
  Strings prefixes;
+ OutputsSpec outputsSpec;
  const flake::LockFlags & lockFlags;
  mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
@@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
  ref<EvalState> state,
  FlakeRef && flakeRef,
  std::string_view fragment,
+ OutputsSpec outputsSpec,
  Strings attrPaths,
  Strings prefixes,
  const flake::LockFlags & lockFlags);

src/libexpr/eval-cache.cc

@@ -47,7 +47,7 @@ struct AttrDb
  {
  auto state(_state->lock());
 
- Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+ Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
  createDirs(cacheDir);
 
  Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -175,6 +175,24 @@ struct AttrDb
  });
  }
 
+ AttrId setListOfStrings(
+ AttrKey key,
+ const std::vector<std::string> & l)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (symbols[key.second])
+ (AttrType::ListOfStrings)
+ (concatStringsSep("\t", l)).exec();
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
  AttrId setPlaceholder(AttrKey key)
  {
  return doSQLite([&]()
@@ -269,6 +287,8 @@ struct AttrDb
  }
  case AttrType::Bool:
  return {{rowId, queryAttribute.getInt(2) != 0}};
+ case AttrType::ListOfStrings:
+ return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
  case AttrType::Missing:
  return {{rowId, missing_t()}};
  case AttrType::Misc:
@@ -385,7 +405,7 @@ std::string AttrCursor::getAttrPathStr(Symbol name) const
 
 Value & AttrCursor::forceValue()
 {
- debug("evaluating uncached attribute %s", getAttrPathStr());
+ debug("evaluating uncached attribute '%s'", getAttrPathStr());
 
  auto & v = getValue();
 
@@ -626,6 +646,38 @@ bool AttrCursor::getBool()
  return v.boolean;
 }
 
+std::vector<std::string> AttrCursor::getListOfStrings()
+{
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = root->db->getAttr(getKey());
+ if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+ if (auto l = std::get_if<std::vector<std::string>>(&cachedValue->second)) {
+ debug("using cached list of strings attribute '%s'", getAttrPathStr());
+ return *l;
+ } else
+ throw TypeError("'%s' is not a list of strings", getAttrPathStr());
+ }
+ }
+
+ debug("evaluating uncached attribute '%s'", getAttrPathStr());
+
+ auto & v = getValue();
+ root->state.forceValue(v, noPos);
+
+ if (v.type() != nList)
+ throw TypeError("'%s' is not a list", getAttrPathStr());
+
+ std::vector<std::string> res;
+
+ for (auto & elem : v.listItems())
+ res.push_back(std::string(root->state.forceStringNoCtx(*elem)));
+
+ cachedValue = {root->db->setListOfStrings(getKey(), res), res};
+
+ return res;
+}
+
 std::vector<Symbol> AttrCursor::getAttrs()
 {
  if (root->db) {

src/libexpr/eval-cache.hh

@@ -44,6 +44,7 @@ enum AttrType {
  Misc = 4,
  Failed = 5,
  Bool = 6,
+ ListOfStrings = 7,
 };
 
 struct placeholder_t {};
@@ -61,7 +62,8 @@ typedef std::variant<
  missing_t,
  misc_t,
  failed_t,
- bool
+ bool,
+ std::vector<std::string>
  > AttrValue;
 
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
@@ -114,6 +116,8 @@ public:
 
  bool getBool();
 
+ std::vector<std::string> getListOfStrings();
+
  std::vector<Symbol> getAttrs();
 
  bool isDerivation();

src/libexpr/flake/config.cc

@@ -50,13 +50,11 @@ void ConfigFile::apply()
  else
  assert(false);
 
- if (!whitelist.count(baseName)) {
- auto trustedList = readTrustedList();
-
+ if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
  bool trusted = false;
- if (nix::fetchSettings.acceptFlakeConfig){
-  trusted = true;
- } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+ auto trustedList = readTrustedList();
+ auto tlname = get(trustedList, name);
+ if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
  trusted = *saved;
  warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
  } else {
@@ -69,7 +67,6 @@ void ConfigFile::apply()
  writeTrustedList(trustedList);
  }
  }
-
  if (!trusted) {
  warn("ignoring untrusted flake configuration setting '%s'", name);
  continue;

src/libexpr/flake/flakeref.cc

@@ -176,7 +176,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
  parsedURL.query.insert_or_assign("shallow", "1");
 
  return std::make_pair(
- FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+ FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
  fragment);
  }
 
@@ -189,7 +189,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
  if (!hasPrefix(path, "/"))
  throw BadURL("flake reference '%s' is not an absolute path", url);
  auto query = decodeQuery(match[2]);
- path = canonPath(path + "/" + get(query, "dir").value_or(""));
+ path = canonPath(path + "/" + getOr(query, "dir", ""));
  }
 
  fetchers::Attrs attrs;
@@ -208,7 +208,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
  input.parent = baseDir;
 
  return std::make_pair(
- FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
+ FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
  fragment);
  }
 }
@@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
  return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
 }
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+ const std::string & url,
+ const std::optional<Path> & baseDir,
+ bool allowMissing,
+ bool isFlake)
+{
+ auto [prefix, outputsSpec] = parseOutputsSpec(url);
+ auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
+ return {std::move(flakeRef), fragment, outputsSpec};
+}
+
 }