Make ValidPath (nar) hash optional if ca field is present

perl/lib/Nix/Store.xs

@@ -80,8 +80,13 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
  PPCODE:
  try {
- auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
- XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
+ auto storePath = store()->parseStorePath(path);
+ if (auto optHash = store()->queryPathInfo(storePath)->optNarHash()) {
+ auto s = optHash->to_string(Base32, true);
+ XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
+ } else {
+ XPUSHs((SV *) NULL);
+ }
  } catch (Error & e) {
  croak("%s", e.what());
  }
@@ -106,10 +111,17 @@ SV * queryPathInfo(char * path, int base32)
  XPUSHs(&PL_sv_undef);
  else
  XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
- auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
- XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
- mXPUSHi(info->registrationTime);
- mXPUSHi(info->narSize);
+ if (auto optHashSize = *info->viewHashResultConst()) {
+ auto [narHash, narSize] = *optHashSize;
+ auto s = narHash.to_string(base32 ? Base32 : Base16, true);
+ XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
+ mXPUSHi(info->registrationTime);
+ mXPUSHi(narSize);
+ } else {
+ XPUSHs((SV *) NULL);
+ mXPUSHi(info->registrationTime);
+ mXPUSHi(0);
+ }
  AV * arr = newAV();
  for (auto & i : info->references)
  av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));

src/libexpr/primops/fetchTree.cc

@@ -207,8 +207,11 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
  auto path = state.store->toRealPath(storePath);
 
  if (expectedHash) {
+ auto narHashResult = *state.store->queryPathInfo(storePath)->viewHashResultConst();
+ assert(narHashResult);
+ auto narHash = narHashResult->first;
  auto hash = unpack
- ? state.store->queryPathInfo(storePath)->narHash
+ ? narHash
  : hashFile(htSHA256, path);
  if (hash != *expectedHash)
  throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",

src/libexpr/value-to-xml.hh

@@ -10,5 +10,5 @@ namespace nix {
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
  Value & v, std::ostream & out, PathSet & context);
- 
+
 }

src/libfetchers/fetchers.cc

@@ -137,13 +137,15 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
  if (tree.actualPath == "")
  tree.actualPath = store->toRealPath(tree.storePath);
 
- auto narHash = store->queryPathInfo(tree.storePath)->narHash;
- input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
-
- if (auto prevNarHash = getNarHash()) {
- if (narHash != *prevNarHash)
- throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+ if (auto optNarHash = store->queryPathInfo(tree.storePath)->optNarHash()) {
+ auto narHash = *optNarHash;
+ input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
+
+ if (auto prevNarHash = getNarHash()) {
+ if (narHash != *prevNarHash)
+ throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+ }
  }
 
  if (auto prevLastModified = getLastModified()) {

src/libfetchers/tarball.cc

@@ -72,12 +72,10 @@ DownloadFileResult downloadFile(
  auto hash = hashString(htSHA256, *res.data);
  ValidPathInfo info {
  store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
- hashString(htSHA256, *sink.s),
- };
- info.narSize = sink.s->size();
- info.ca = FixedOutputHash {
- .method = FileIngestionMethod::Flat,
- .hash = hash,
+ std::pair<HashResult, ContentAddress> {
+ { hashString(htSHA256, *sink.s), sink.s->size() },
+ FixedOutputHash { .method = FileIngestionMethod::Flat, .hash = hash },
+ },
  };
  auto source = StringSource { *sink.s };
  store->addToStore(info, source, NoRepair, NoCheckSigs);

src/libstore/binary-cache-store.cc

@@ -183,10 +183,11 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
  compression == "br" ? ".br" :
  "");
 
+ auto narSize = *narInfo->optNarSize();
  auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
  printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
- printStorePath(narInfo->path), info.narSize,
- ((1.0 - (double) fileSize / info.narSize) * 100.0),
+ printStorePath(narInfo->path), narSize,
+ ((1.0 - (double) fileSize / narSize) * 100.0),
  duration);
 
  /* Verify that all references are valid. This may do some .narinfo
@@ -284,7 +285,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
  } else
  stats.narWriteAverted++;
 
- stats.narWriteBytes += info.narSize;
+ stats.narWriteBytes += narSize;
  stats.narWriteCompressedBytes += fileSize;
  stats.narWriteCompressionTimeMs += duration;
 
@@ -324,9 +325,14 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
  return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
  ValidPathInfo info {
  makeFixedOutputPath(method, nar.first, name),
- nar.first,
+ (ContentAddresses) std::pair {
+ nar,
+ (ContentAddress) FixedOutputHash {
+ .method = method,
+ .hash = nar.first,
+ },
+ },
  };
- info.narSize = nar.second;
  return info;
  })->path;
 }
@@ -414,12 +420,13 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
  return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
  ValidPathInfo info {
  makeFixedOutputPath(method, h, name),
- nar.first,
- };
- info.narSize = nar.second;
- info.ca = FixedOutputHash {
- .method = method,
- .hash = h,
+ (ContentAddresses) std::pair {
+ nar,
+ (ContentAddress) FixedOutputHash {
+ .method = method,
+ .hash = h,
+ },
+ },
  };
  return info;
  })->path;
@@ -436,9 +443,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
 
  auto source = StringSource { s };
  return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
- ValidPathInfo info { path, nar.first };
- info.narSize = nar.second;
- info.ca = TextHash { textHash };
+ ValidPathInfo info {
+ path,
+ (ContentAddresses) std::pair {
+ nar,
+ (ContentAddress) TextHash { textHash },
+ },
+ };
  info.references = references;
  return info;
  })->path;

src/libstore/build/derivation-goal.cc

@@ -3101,18 +3101,19 @@ void DerivationGoal::registerOutputs()
  outputPathName(drv->name, outputName),
  refs.second,
  refs.first),
- narHashAndSize.first,
+ This<HashResult> { narHashAndSize },
  };
- newInfo0.narSize = narHashAndSize.second;
- newInfo0.ca = FixedOutputHash {
- .method = outputHash.method,
- .hash = got,
+ newInfo0.viewCA() = ContentAddress {
+ FixedOutputHash {
+ .method = outputHash.method,
+ .hash = got,
+ }
  };
  newInfo0.references = refs.second;
  if (refs.first)
  newInfo0.references.insert(newInfo0.path);
 
- assert(newInfo0.ca);
+ assert(newInfo0.optCA());
  return newInfo0;
  };
 
@@ -3128,8 +3129,7 @@ void DerivationGoal::registerOutputs()
  std::string { requiredFinalPath.hashPart() });
  rewriteOutput();
  auto narHashAndSize = hashPath(htSHA256, actualPath);
- ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
- newInfo0.narSize = narHashAndSize.second;
+ ValidPathInfo newInfo0 { requiredFinalPath, This<HashResult> { narHashAndSize } };
  auto refs = rewriteRefs();
  newInfo0.references = refs.second;
  if (refs.first)
@@ -3144,8 +3144,8 @@ void DerivationGoal::registerOutputs()
 
  /* Check wanted hash */
  Hash & wanted = dof.hash.hash;
- assert(newInfo0.ca);
- auto got = getContentAddressHash(*newInfo0.ca);
+ assert(newInfo0.optCA());
+ auto got = getContentAddressHash(*newInfo0.optCA());
  if (wanted != got) {
  /* Throw an error after registering the path as
  valid. */
@@ -3163,6 +3163,9 @@ void DerivationGoal::registerOutputs()
  },
  }, output.output);
 
+ /* Always calculate nar hash for new builds. */
+ assert(newInfo.optNarHash().has_value());
+
  /* Calculate where we'll move the output files. In the checking case we
  will leave leave them where they are, for now, rather than move to
  their usual "final destination" */
@@ -3176,7 +3179,7 @@ void DerivationGoal::registerOutputs()
  if (!optFixedPath ||
  worker.store.printStorePath(*optFixedPath) != finalDestPath)
  {
- assert(newInfo.ca);
+ assert(newInfo.optCA());
  dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
  }
 
@@ -3192,7 +3195,7 @@ void DerivationGoal::registerOutputs()
  } else if (worker.store.isValidPath(newInfo.path)) {
  /* Path already exists because CA path produced by something
  else. No moving needed. */
- assert(newInfo.ca);
+ assert(newInfo.optCA());
  } else {
  auto destPath = worker.store.toRealPath(finalDestPath);
  movePath(actualPath, destPath);
@@ -3203,7 +3206,7 @@ void DerivationGoal::registerOutputs()
  if (buildMode == bmCheck) {
  if (!worker.store.isValidPath(newInfo.path)) continue;
  ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
- if (newInfo.narHash != oldInfo.narHash) {
+ if (newInfo.optNarHash() != oldInfo.optNarHash()) {
  worker.checkMismatch = true;
  if (settings.runDiffHook || settings.keepFailed) {
  auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
@@ -3255,7 +3258,7 @@ void DerivationGoal::registerOutputs()
  /* If it's a CA path, register it right away. This is necessary if it
  isn't statically known so that we can safely unlock the path before
  the next iteration */
- if (newInfo.ca)
+ if (newInfo.optCA().has_value())
  worker.store.registerValidPaths({newInfo});
 
  infos.emplace(outputName, std::move(newInfo));
@@ -3396,16 +3399,20 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
  pathsLeft.pop();
  if (!pathsDone.insert(path).second) continue;
 
+ auto accumPath = [&](const ValidPathInfo & info) {
+ std::optional optNarSize = info.optNarSize();
+ assert(optNarSize); // Always require NAR size for path in local store;
+ closureSize += *optNarSize;
+ for (auto & ref : info.references)
+ pathsLeft.push(ref);
+ };
+
  auto i = outputsByPath.find(worker.store.printStorePath(path));
  if (i != outputsByPath.end()) {
- closureSize += i->second.narSize;
- for (auto & ref : i->second.references)
- pathsLeft.push(ref);
+ accumPath(i->second);
  } else {
  auto info = worker.store.queryPathInfo(path);
- closureSize += info->narSize;
- for (auto & ref : info->references)
- pathsLeft.push(ref);
+ accumPath(*info);
  }
  }
 
@@ -3414,9 +3421,12 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 
  auto applyChecks = [&](const Checks & checks)
  {
- if (checks.maxSize && info.narSize > *checks.maxSize)
+ std::optional optNarSize = info.optNarSize();
+ assert(optNarSize); // Always require NAR size for path in local store;
+ auto narSize = *optNarSize;
+ if (checks.maxSize && narSize > *checks.maxSize)
  throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
- worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
+ worker.store.printStorePath(info.path), narSize, *checks.maxSize);
 
  if (checks.maxClosureSize) {
  uint64_t closureSize = getClosure(info.path).second;

src/libstore/build/substitution-goal.cc

@@ -130,7 +130,11 @@ void SubstitutionGoal::tryNext()
  /* Update the total expected download size. */
  auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
- maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
+ auto narSizeOpt = info->optNarSize();
+ maintainExpectedNar =
+ narSizeOpt
+ ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, *narSizeOpt)
+ : nullptr;
 
  maintainExpectedDownload =
  narInfo && narInfo->fileSize
@@ -274,7 +278,8 @@ void SubstitutionGoal::finished()
  worker.doneDownloadSize += fileSize;
  }
 
- worker.doneNarSize += maintainExpectedNar->delta;
+ if (maintainExpectedNar)
+ worker.doneNarSize += maintainExpectedNar->delta;
  maintainExpectedNar.reset();
 
  worker.updateProgress();

src/libstore/build/worker.cc

@@ -433,9 +433,10 @@ bool Worker::pathContentsGood(const StorePath & path)
  if (!pathExists(store.printStorePath(path)))
  res = false;
  else {
- HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
+ Hash narHash = (*info->viewHashResultConst())->first;
+ HashResult current = hashPath(narHash.type, store.printStorePath(path));
  Hash nullHash(htSHA256);
- res = info->narHash == nullHash || info->narHash == current.first;
+ res = narHash == nullHash || narHash == current.first;
  }
  pathContentsGoodCache.insert_or_assign(path, res);
  if (!res)

src/libstore/content-address.cc

@@ -4,6 +4,24 @@
 
 namespace nix {
 
+bool TextHash::operator ==(const TextHash & otherHash) const noexcept {
+ return hash == otherHash.hash;
+};
+
+bool TextHash::operator !=(const TextHash & otherHash) const noexcept {
+ return hash != otherHash.hash;
+};
+
+
+bool FixedOutputHash::operator ==(const FixedOutputHash & otherHash) const noexcept {
+ return method == otherHash.method && hash == otherHash.hash;
+};
+
+bool FixedOutputHash::operator !=(const FixedOutputHash & otherHash) const noexcept {
+ return method != otherHash.method || hash != otherHash.hash;
+};
+
+
 std::string FixedOutputHash::printMethodAlgo() const
 {
  return makeFileIngestionPrefix(method) + printHashType(hash.type);

src/libstore/content-address.hh

@@ -12,12 +12,16 @@ enum struct FileIngestionMethod : uint8_t {
 
 struct TextHash {
  Hash hash;
+ bool operator ==(const TextHash & otherHash) const noexcept;
+ bool operator !=(const TextHash & otherHash) const noexcept;
 };
 
 /// Pair of a hash, and how the file system was ingested
 struct FixedOutputHash {
  FileIngestionMethod method;
  Hash hash;
+ bool operator ==(const FixedOutputHash & therHash) const noexcept;
+ bool operator !=(const FixedOutputHash & therHash) const noexcept;
  std::string printMethodAlgo() const;
 };