Merge pull request #9452 from kolloch/feature/nix-hash-convert

Add nix hash convert

doc/manual/rl-next/hash-format-nix32.md

@@ -0,0 +1,22 @@
+synopsis: Rename hash format `base32` to `nix32`
+prs: #9452
+description: {
+
+Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for 
+[Base32](https://en.wikipedia.org/wiki/Base32).
+
+## Deprecation: Use `nix32` instead of `base32` as `toHashFormat`
+
+For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from`
+parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value
+remains as a deprecated alias for `"base32"`. Please convert your code from:
+
+```nix
+builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}
+```
+
+to
+
+```nix
+builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}
+```

doc/manual/rl-next/nix-hash-convert.md

@@ -0,0 +1,47 @@
+synopsis: Add `nix hash convert`
+prs: #9452
+description: {
+
+New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track
+to stabilization! Examples:
+
+- Convert the hash to `nix32`.
+
+ ```bash
+ $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+ vw46m23bizj4n8afrc0fj19wrp7mj3c0
+ ```
+ `nix32` is a base32 encoding with a nix-specific character set.
+ Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input
+ hash.
+- Convert the hash to the `sri` format that includes an algorithm specification:
+ ```bash
+ nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+ sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
+ ```
+ or with an explicit `-to` format:
+ ```bash
+ nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+ sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
+ ```
+- Assert the input format of the hash:
+ ```bash
+ nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
+ error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32'
+ nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+ sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=
+ ```
+
+The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion.
+
+## Related Deprecations
+
+The following commands are still available but will emit a deprecation warning. Please convert your code to
+`nix hash convert`:
+
+- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead.
+- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead.
+- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
+- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2`
+ or even just `nix hash convert $hash1 $hash2` instead.
+}

perl/lib/Nix/Store.xs

@@ -78,7 +78,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
  PPCODE:
  try {
- auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true);
+ auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true);
  XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
  } catch (Error & e) {
  croak("%s", e.what());
@@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32)
  XPUSHs(&PL_sv_undef);
  else
  XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
- auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true);
+ auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true);
  XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
  mXPUSHi(info->registrationTime);
  mXPUSHi(info->narSize);
@@ -205,8 +205,8 @@ void importPaths(int fd, int dontCheckSigs)
 SV * hashPath(char * algo, int base32, char * path)
  PPCODE:
  try {
- Hash h = hashPath(parseHashType(algo), path).first;
- auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+ Hash h = hashPath(parseHashAlgo(algo), path).first;
+ auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
  XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
  } catch (Error & e) {
  croak("%s", e.what());
@@ -216,8 +216,8 @@ SV * hashPath(char * algo, int base32, char * path)
 SV * hashFile(char * algo, int base32, char * path)
  PPCODE:
  try {
- Hash h = hashFile(parseHashType(algo), path);
- auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+ Hash h = hashFile(parseHashAlgo(algo), path);
+ auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
  XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
  } catch (Error & e) {
  croak("%s", e.what());
@@ -227,8 +227,8 @@ SV * hashFile(char * algo, int base32, char * path)
 SV * hashString(char * algo, int base32, char * s)
  PPCODE:
  try {
- Hash h = hashString(parseHashType(algo), s);
- auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+ Hash h = hashString(parseHashAlgo(algo), s);
+ auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
  XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
  } catch (Error & e) {
  croak("%s", e.what());
@@ -238,8 +238,8 @@ SV * hashString(char * algo, int base32, char * s)
 SV * convertHash(char * algo, char * s, int toBase32)
  PPCODE:
  try {
- auto h = Hash::parseAny(s, parseHashType(algo));
- auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
+ auto h = Hash::parseAny(s, parseHashAlgo(algo));
+ auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false);
  XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
  } catch (Error & e) {
  croak("%s", e.what());
@@ -281,7 +281,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
  PPCODE:
  try {
  auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
+ auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo));
  XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
  } catch (Error & e) {
  croak("%s", e.what());
@@ -291,7 +291,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
 SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
  PPCODE:
  try {
- auto h = Hash::parseAny(hash, parseHashType(algo));
+ auto h = Hash::parseAny(hash, parseHashAlgo(algo));
  auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
  auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
  .method = method,

src/libexpr/flake/flake.cc

@@ -904,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const
  // FIXME: as an optimization, if the flake contains a lock file
  // and we haven't changed it, then it's sufficient to use
  // flake.sourceInfo.storePath for the fingerprint.
- return hashString(htSHA256,
+ return hashString(HashAlgorithm::SHA256,
  fmt("%s;%s;%d;%d;%s",
  flake.storePath.to_string(),
  flake.lockedRef.subdir,

src/libexpr/primops.cc

@@ -1317,7 +1317,7 @@ drvName, Bindings * attrs, Value & v)
  .errPos = state.positions[noPos]
  }));
 
- auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
+ auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo));
 
  auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
 
@@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v)
  .errPos = state.positions[noPos]
  });
 
- auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
+ auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
  auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
  for (auto & i : outputs) {
@@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v)
  drv.outputs.insert_or_assign(i,
  DerivationOutput::Impure {
  .method = method,
- .hashType = ht,
+ .hashAlgo = ha,
  });
  else
  drv.outputs.insert_or_assign(i,
  DerivationOutput::CAFloating {
  .method = method,
- .hashType = ht,
+ .hashAlgo = ha,
  });
  }
  }
@@ -1754,17 +1754,17 @@ static RegisterPrimOp primop_findFile(PrimOp {
 /* Return the cryptographic hash of a file in base-16. */
 static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
- auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
- std::optional<HashType> ht = parseHashType(type);
- if (!ht)
+ auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
+ std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
+ if (!ha)
  state.debugThrowLastTrace(Error({
- .msg = hintfmt("unknown hash type '%1%'", type),
+ .msg = hintfmt("unknown hash algo '%1%'", algo),
  .errPos = state.positions[pos]
  }));
 
  auto path = realisePath(state, pos, *args[1]);
 
- v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false));
+ v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2341,7 +2341,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
  else if (n == "recursive")
  method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
  else if (n == "sha256")
- expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256);
+ expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
  else
  state.debugThrowLastTrace(EvalError({
  .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]),
@@ -3766,18 +3766,18 @@ static RegisterPrimOp primop_stringLength({
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
- auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
- std::optional<HashType> ht = parseHashType(type);
- if (!ht)
+ auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
+ std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
+ if (!ha)
  state.debugThrowLastTrace(Error({
- .msg = hintfmt("unknown hash type '%1%'", type),
+ .msg = hintfmt("unknown hash algo '%1%'", algo),
  .errPos = state.positions[pos]
  }));
 
  NixStringContext context; // discarded
  auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
- v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false));
+ v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -3800,15 +3800,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args
  auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
 
  Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo"));
- std::optional<HashType> ht = std::nullopt;
+ std::optional<HashAlgorithm> ha = std::nullopt;
  if (iteratorHashAlgo != inputAttrs->end()) {
- ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
+ ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
  }
 
  Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'");
  HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
- v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI));
+ v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
 }
 
 static RegisterPrimOp primop_convertHash({
@@ -3837,7 +3837,8 @@ static RegisterPrimOp primop_convertHash({
 
  The format of the resulting hash. Must be one of
  - `"base16"`
- - `"base32"`
+ - `"nix32"`
+ - `"base32"` (deprecated alias for `"nix32"`)
  - `"base64"`
  - `"sri"`
 

src/libexpr/primops/fetchMercurial.cc

@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
  // be both a revision or a branch/tag name.
  auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial");
  if (std::regex_match(value.begin(), value.end(), revRegex))
- rev = Hash::parseAny(value, htSHA1);
+ rev = Hash::parseAny(value, HashAlgorithm::SHA1);
  else
  ref = value;
  }
@@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
  attrs2.alloc("branch").mkString(*input2.getRef());
  // Backward compatibility: set 'rev' to
  // 0000000000000000000000000000000000000000 for a dirty tree.
- auto rev2 = input2.getRev().value_or(Hash(htSHA1));
+ auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
  attrs2.alloc("rev").mkString(rev2.gitRev());
  attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
  if (auto revCount = input2.getRevCount())

src/libexpr/primops/fetchTree.cc

@@ -46,7 +46,7 @@ void emitTreeAttrs(
  attrs.alloc("shortRev").mkString(rev->gitShortRev());
  } else if (emptyRevFallback) {
  // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
- auto emptyHash = Hash(htSHA1);
+ auto emptyHash = Hash(HashAlgorithm::SHA1);
  attrs.alloc("rev").mkString(emptyHash.gitRev());
  attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
  }
@@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
  if (n == "url")
  url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
  else if (n == "sha256")
- expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256);
+ expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
  else if (n == "name")
  name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
  else
@@ -276,7 +276,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
  state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
 
  // early exit if pinned and already in the store
- if (expectedHash && expectedHash->type == htSHA256) {
+ if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) {
  auto expectedPath = state.store->makeFixedOutputPath(
  name,
  FixedOutputInfo {
@@ -301,10 +301,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
  if (expectedHash) {
  auto hash = unpack
  ? state.store->queryPathInfo(storePath)->narHash
- : hashFile(htSHA256, state.store->toRealPath(storePath));
+ : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
  if (hash != *expectedHash)
  state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
- *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
+  *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)));
  }
 
  state.allowAndSetStorePathString(storePath, v);

src/libfetchers/fetchers.cc

@@ -289,8 +289,8 @@ std::string Input::getType() const
 std::optional<Hash> Input::getNarHash() const
 {
  if (auto s = maybeGetStrAttr(attrs, "narHash")) {
- auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
- if (hash.type != htSHA256)
+ auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s);
+ if (hash.algo != HashAlgorithm::SHA256)
  throw UsageError("narHash must use SHA-256");
  return hash;
  }
@@ -314,7 +314,7 @@ std::optional<Hash> Input::getRev() const
  } catch (BadHash &e) {
  // Default to sha1 for backwards compatibility with existing
  // usages (e.g. `builtins.fetchTree` calls or flake inputs).
- hash = Hash::parseAny(*s, htSHA1);
+ hash = Hash::parseAny(*s, HashAlgorithm::SHA1);
  }
  }
 

src/libfetchers/git-utils.cc

@@ -91,7 +91,7 @@ Hash toHash(const git_oid & oid)
  #ifdef GIT_EXPERIMENTAL_SHA256
  assert(oid.type == GIT_OID_SHA1);
  #endif
- Hash hash(htSHA1);
+ Hash hash(HashAlgorithm::SHA1);
  memcpy(hash.hash, oid.id, hash.hashSize);
  return hash;
 }
@@ -439,7 +439,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
  std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
  for (const fetchers::PublicKey & k : publicKeys){
  // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
- auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+ auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
  auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
  re += "(" + escaped_fingerprint + ")";
  }