Make ValidPath (nar) hash optional if ca field is present

src/libexpr/primops/fetchTree.cc

@@ -207,8 +207,11 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
  auto path = state.store->toRealPath(storePath);
 
  if (expectedHash) {
+ auto narHashResult = *state.store->queryPathInfo(storePath)->viewHashResultConst();
+ assert(narHashResult);
+ auto narHash = narHashResult->first;
  auto hash = unpack
- ? state.store->queryPathInfo(storePath)->narHash
+ ? narHash
  : hashFile(htSHA256, path);
  if (hash != *expectedHash)
  throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",

src/libexpr/value-to-xml.hh

@@ -10,5 +10,5 @@ namespace nix {
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
  Value & v, std::ostream & out, PathSet & context);
- 
+
 }

src/libfetchers/fetchers.cc

@@ -129,13 +129,15 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
  if (tree.actualPath == "")
  tree.actualPath = store->toRealPath(tree.storePath);
 
- auto narHash = store->queryPathInfo(tree.storePath)->narHash;
- input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
-
- if (auto prevNarHash = getNarHash()) {
- if (narHash != *prevNarHash)
- throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+ if (auto optNarHash = store->queryPathInfo(tree.storePath)->optNarHash()) {
+ auto narHash = *optNarHash;
+ input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
+
+ if (auto prevNarHash = getNarHash()) {
+ if (narHash != *prevNarHash)
+ throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+ }
  }
 
  if (auto prevLastModified = getLastModified()) {

src/libfetchers/tarball.cc

@@ -69,12 +69,10 @@ DownloadFileResult downloadFile(
  auto hash = hashString(htSHA256, *res.data);
  ValidPathInfo info {
  store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
- hashString(htSHA256, *sink.s),
- };
- info.narSize = sink.s->size();
- info.ca = FixedOutputHash {
- .method = FileIngestionMethod::Flat,
- .hash = hash,
+ std::pair<HashResult, ContentAddress> {
+ { hashString(htSHA256, *sink.s), sink.s->size() },
+ FixedOutputHash { .method = FileIngestionMethod::Flat, .hash = hash },
+ },
  };
  auto source = StringSource { *sink.s };
  store->addToStore(info, source, NoRepair, NoCheckSigs);

src/libstore/binary-cache-store.cc

@@ -145,7 +145,8 @@ struct FileSource : FdSource
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
  RepairFlag repair, CheckSigsFlag checkSigs)
 {
- assert(info.narSize);
+ assert(info.optNarSize());
+ auto narSize = *info.optNarSize();
 
  if (!repair && isValidPath(info.path)) {
  // FIXME: copyNAR -> null sink
@@ -177,8 +178,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
  auto now2 = std::chrono::steady_clock::now();
 
  auto narInfo = make_ref<NarInfo>(info);
- narInfo->narSize = info.narSize;
- narInfo->narHash = info.narHash;
  narInfo->compression = compression;
  auto [fileHash, fileSize] = fileHashSink.finish();
  narInfo->fileHash = fileHash;
@@ -191,8 +190,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
  auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
  printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
- printStorePath(narInfo->path), info.narSize,
- ((1.0 - (double) fileSize / info.narSize) * 100.0),
+ printStorePath(narInfo->path), narSize,
+ ((1.0 - (double) fileSize / narSize) * 100.0),
  duration);
 
  /* Verify that all references are valid. This may do some .narinfo
@@ -290,7 +289,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
  } else
  stats.narWriteAverted++;
 
- stats.narWriteBytes += info.narSize;
+ stats.narWriteBytes += narSize;
  stats.narWriteCompressedBytes += fileSize;
  stats.narWriteCompressionTimeMs += duration;
 
@@ -385,7 +384,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
 
  ValidPathInfo info {
  makeFixedOutputPath(method, *h, name),
- Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+ This<HashResult> { { Hash::dummy, 0 } }, // Will be fixed in addToStore, which recomputes nar hash
  };
 
  auto source = StringSource { *sink.s };
@@ -399,7 +398,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
 {
  ValidPathInfo info {
  computeStorePathForText(name, s, references),
- Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+ This<HashResult> { { Hash::dummy, 0 } }, // Will be fixed in addToStore, which recomputes nar hash
  };
  info.references = references;
 

src/libstore/build.cc

@@ -4010,18 +4010,19 @@ void DerivationGoal::registerOutputs()
  outputPathName(drv->name, outputName),
  refs.second,
  refs.first),
- narHashAndSize.first,
+ This<HashResult> { narHashAndSize },
  };
- newInfo0.narSize = narHashAndSize.second;
- newInfo0.ca = FixedOutputHash {
- .method = outputHash.method,
- .hash = got,
+ newInfo0.viewCA() = ContentAddress {
+ FixedOutputHash {
+ .method = outputHash.method,
+ .hash = got,
+ }
  };
  newInfo0.references = refs.second;
  if (refs.first)
  newInfo0.references.insert(newInfo0.path);
 
- assert(newInfo0.ca);
+ assert(newInfo0.optCA());
  return newInfo0;
  };
 
@@ -4037,8 +4038,7 @@ void DerivationGoal::registerOutputs()
  std::string { requiredFinalPath.hashPart() });
  rewriteOutput();
  auto narHashAndSize = hashPath(htSHA256, actualPath);
- ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
- newInfo0.narSize = narHashAndSize.second;
+ ValidPathInfo newInfo0 { requiredFinalPath, This<HashResult> { narHashAndSize } };
  auto refs = rewriteRefs();
  newInfo0.references = refs.second;
  if (refs.first)
@@ -4053,8 +4053,8 @@ void DerivationGoal::registerOutputs()
 
  /* Check wanted hash */
  Hash & wanted = dof.hash.hash;
- assert(newInfo0.ca);
- auto got = getContentAddressHash(*newInfo0.ca);
+ assert(newInfo0.optCA());
+ auto got = getContentAddressHash(*newInfo0.optCA());
  if (wanted != got) {
  /* Throw an error after registering the path as
  valid. */
@@ -4072,6 +4072,9 @@ void DerivationGoal::registerOutputs()
  },
  }, output.output);
 
+ /* Always calculate nar hash for new builds. */
+ assert(newInfo.optNarHash().has_value());
+
  /* Calculate where we'll move the output files. In the checking case we
  will leave leave them where they are, for now, rather than move to
  their usual "final destination" */
@@ -4085,7 +4088,7 @@ void DerivationGoal::registerOutputs()
  if (!optFixedPath ||
  worker.store.printStorePath(*optFixedPath) != finalDestPath)
  {
- assert(newInfo.ca);
+ assert(newInfo.optCA());
  dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
  }
 
@@ -4101,7 +4104,7 @@ void DerivationGoal::registerOutputs()
  } else if (worker.store.isValidPath(newInfo.path)) {
  /* Path already exists because CA path produced by something
  else. No moving needed. */
- assert(newInfo.ca);
+ assert(newInfo.optCA());
  } else {
  auto destPath = worker.store.toRealPath(finalDestPath);
  movePath(actualPath, destPath);
@@ -4112,7 +4115,7 @@ void DerivationGoal::registerOutputs()
  if (buildMode == bmCheck) {
  if (!worker.store.isValidPath(newInfo.path)) continue;
  ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
- if (newInfo.narHash != oldInfo.narHash) {
+ if (newInfo.optNarHash() != oldInfo.optNarHash()) {
  worker.checkMismatch = true;
  if (settings.runDiffHook || settings.keepFailed) {
  auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
@@ -4164,7 +4167,7 @@ void DerivationGoal::registerOutputs()
  /* If it's a CA path, register it right away. This is necessary if it
  isn't statically known so that we can safely unlock the path before
  the next iteration */
- if (newInfo.ca)
+ if (newInfo.optCA().has_value())
  worker.store.registerValidPaths({newInfo});
 
  infos.emplace(outputName, std::move(newInfo));
@@ -4294,16 +4297,20 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
  pathsLeft.pop();
  if (!pathsDone.insert(path).second) continue;
 
+ auto accumPath = [&](const ValidPathInfo & info) {
+ std::optional optNarSize = info.optNarSize();
+ assert(optNarSize); // Always require NAR size for path in local store;
+ closureSize += *optNarSize;
+ for (auto & ref : info.references)
+ pathsLeft.push(ref);
+ };
+
  auto i = outputsByPath.find(worker.store.printStorePath(path));
  if (i != outputsByPath.end()) {
- closureSize += i->second.narSize;
- for (auto & ref : i->second.references)
- pathsLeft.push(ref);
+ accumPath(i->second);
  } else {
  auto info = worker.store.queryPathInfo(path);
- closureSize += info->narSize;
- for (auto & ref : info->references)
- pathsLeft.push(ref);
+ accumPath(*info);
  }
  }
 
@@ -4312,9 +4319,12 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 
  auto applyChecks = [&](const Checks & checks)
  {
- if (checks.maxSize && info.narSize > *checks.maxSize)
+ std::optional optNarSize = info.optNarSize();
+ assert(optNarSize); // Always require NAR size for path in local store;
+ auto narSize = *optNarSize;
+ if (checks.maxSize && narSize > *checks.maxSize)
  throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
- worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
+ worker.store.printStorePath(info.path), narSize, *checks.maxSize);
 
  if (checks.maxClosureSize) {
  uint64_t closureSize = getClosure(info.path).second;
@@ -4843,7 +4853,11 @@ void SubstitutionGoal::tryNext()
  /* Update the total expected download size. */
  auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
- maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
+ auto narSizeOpt = info->optNarSize();
+ maintainExpectedNar =
+ narSizeOpt
+ ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, *narSizeOpt)
+ : nullptr;
 
  maintainExpectedDownload =
  narInfo && narInfo->fileSize
@@ -4987,7 +5001,8 @@ void SubstitutionGoal::finished()
  worker.doneDownloadSize += fileSize;
  }
 
- worker.doneNarSize += maintainExpectedNar->delta;
+ if (maintainExpectedNar)
+ worker.doneNarSize += maintainExpectedNar->delta;
  maintainExpectedNar.reset();
 
  worker.updateProgress();
@@ -5417,9 +5432,10 @@ bool Worker::pathContentsGood(const StorePath & path)
  if (!pathExists(store.printStorePath(path)))
  res = false;
  else {
- HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
+ Hash narHash = (*info->viewHashResultConst())->first;
+ HashResult current = hashPath(narHash.type, store.printStorePath(path));
  Hash nullHash(htSHA256);
- res = info->narHash == nullHash || info->narHash == current.first;
+ res = narHash == nullHash || narHash == current.first;
  }
  pathContentsGoodCache.insert_or_assign(path, res);
  if (!res)

src/libstore/content-address.hh

@@ -12,12 +12,24 @@ enum struct FileIngestionMethod : uint8_t {
 
 struct TextHash {
  Hash hash;
+ bool operator ==(TextHash otherHash) const noexcept {
+ return hash == otherHash.hash;
+ };
+ bool operator !=(TextHash otherHash) const noexcept {
+ return hash != otherHash.hash;
+ };
 };
 
 /// Pair of a hash, and how the file system was ingested
 struct FixedOutputHash {
  FileIngestionMethod method;
  Hash hash;
+ bool operator ==(FixedOutputHash otherHash) const noexcept {
+ return method == otherHash.method && hash == otherHash.hash;
+ };
+ bool operator !=(FixedOutputHash otherHash) const noexcept {
+ return method != otherHash.method && hash == otherHash.hash;
+ };
  std::string printMethodAlgo() const;
 };
 

src/libstore/daemon.cc

@@ -245,14 +245,18 @@ static void writeValidPathInfo(
  Sink & to,
  std::shared_ptr<const ValidPathInfo> info)
 {
+ auto narHashResult = *info->viewHashResultConst();
+ assert(narHashResult);
+ auto narHash = narHashResult->first;
+ auto narSize = narHashResult->second;
  to << (info->deriver ? store->printStorePath(*info->deriver) : "")
- << info->narHash.to_string(Base16, false);
+ << narHash.to_string(Base16, false);
  writeStorePaths(*store, to, info->references);
- to << info->registrationTime << info->narSize;
+ to << info->registrationTime << narSize;
  if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
  to << info->ultimate
  << info->sigs
- << renderContentAddress(info->ca);
+ << renderContentAddress(*info->viewCAConst());
  }
 }
 
@@ -303,9 +307,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
  case wopQueryPathHash: {
  auto path = store->parseStorePath(readString(from));
  logger->startWork();
- auto hash = store->queryPathInfo(path)->narHash;
+ auto narHashResult = *store->queryPathInfo(path)->viewHashResultConst();
+ assert(narHashResult);
+ auto narHash = narHashResult->first;
  logger->stopWork();
- to << hash.to_string(Base16, false);
+ to << narHash.to_string(Base16, false);
  break;
  }
 
@@ -779,13 +785,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
  auto path = store->parseStorePath(readString(from));
  auto deriver = readString(from);
  auto narHash = Hash::parseAny(readString(from), htSHA256);
- ValidPathInfo info { path, narHash };
+ ValidPathInfo info { path, This<HashResult> { { narHash, 0 } } };
  if (deriver != "")
  info.deriver = store->parseStorePath(deriver);
  info.references = readStorePaths<StorePathSet>(*store, from);
- from >> info.registrationTime >> info.narSize >> info.ultimate;
+ from >> info.registrationTime;
+ {
+ auto tempNarSize = readInt(from);
+ auto tempHashResult = **info.viewHashResult();
+ info.viewHashResult() = { tempHashResult.first, tempNarSize };
+ }
+ from >> info.ultimate;
  info.sigs = readStrings<StringSet>(from);
- info.ca = parseContentAddressOpt(readString(from));
+ info.viewCA().add(parseContentAddressOpt(readString(from)));
  from >> repair >> dontCheckSigs;
  if (!trusted && dontCheckSigs)
  dontCheckSigs = false;