Add builtins.filterPath

This is like builtins.{filterSource,path}, but returns a virtual path
that applies the filter lazily, rather than copying the result to the
Nix store. Thus filterPath can be composed.

src/libexpr/eval.hh

@@ -525,6 +525,13 @@ public:
  */
  [[nodiscard]] StringMap realiseContext(const PathSet & context);
 
+ /* Call the binary path filter predicate used builtins.path etc. */
+ bool callPathFilter(
+ Value * filterFun,
+ const SourcePath & path,
+ std::string_view pathArg,
+ PosIdx pos);
+
 private:
 
  unsigned long nrEnvs = 0;

src/libexpr/primops.cc

@@ -1974,6 +1974,30 @@ static RegisterPrimOp primop_toFile({
  .fun = prim_toFile,
 });
 
+bool EvalState::callPathFilter(
+ Value * filterFun,
+ const SourcePath & path,
+ std::string_view pathArg,
+ PosIdx pos)
+{
+ auto st = path.lstat();
+
+ /* Call the filter function. The first argument is the path, the
+ second is a string indicating the type of the file. */
+ Value arg1;
+ arg1.mkString(pathArg);
+
+ Value arg2;
+ // assert that type is not "unknown"
+ arg2.mkString(fileTypeToString(st.type));
+
+ Value * args []{&arg1, &arg2};
+ Value res;
+ callFunction(*filterFun, 2, args, res, pos);
+
+ return forceBool(res, pos);
+}
+
 static void addPath(
  EvalState & state,
  const PosIdx pos,
@@ -2010,26 +2034,11 @@ static void addPath(
  #endif
 
  std::unique_ptr<PathFilter> filter;
- if (filterFun) filter = std::make_unique<PathFilter>([&](const Path & p) {
- SourcePath path2{path.accessor, CanonPath(p)};
-
- auto st = path2.lstat();
-
- /* Call the filter function. The first argument is the path,
- the second is a string indicating the type of the file. */
- Value arg1;
- arg1.mkString(path2.path.abs());
-
- Value arg2;
- // assert that type is not "unknown"
- arg2.mkString(fileTypeToString(st.type));
-
- Value * args []{&arg1, &arg2};
- Value res;
- state.callFunction(*filterFun, 2, args, res, pos);
-
- return state.forceBool(res, pos);
- });
+ if (filterFun)
+ filter = std::make_unique<PathFilter>([&](const Path & p) {
+ auto p2 = CanonPath(p);
+ return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
+ });
 
  std::optional<StorePath> expectedStorePath;
  if (expectedHash)
@@ -2130,14 +2139,15 @@ static RegisterPrimOp primop_filterSource({
 
 static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
- state.forceAttrs(*args[0], pos);
  std::optional<SourcePath> path;
  std::string name;
  Value * filterFun = nullptr;
  auto method = FileIngestionMethod::Recursive;
  std::optional<Hash> expectedHash;
  PathSet context;
 
+ state.forceAttrs(*args[0], pos);
+
  for (auto & attr : *args[0]->attrs) {
  auto n = state.symbols[attr.name];
  if (n == "path")

src/libexpr/primops/filterPath.cc

@@ -0,0 +1,174 @@
+#include "primops.hh"
+
+namespace nix {
+
+struct FilteringInputAccessor : InputAccessor
+{
+ EvalState & state;
+ PosIdx pos;
+ ref<InputAccessor> next;
+ CanonPath prefix;
+ Value * filterFun;
+
+ std::map<CanonPath, bool> cache;
+
+ FilteringInputAccessor(EvalState & state, PosIdx pos, const SourcePath & src, Value * filterFun)
+ : state(state)
+ , pos(pos)
+ , next(src.accessor)
+ , prefix(src.path)
+ , filterFun(filterFun)
+ {
+ }
+
+ std::string readFile(const CanonPath & path) override
+ {
+ checkAccess(path);
+ return next->readFile(prefix + path);
+ }
+
+ bool pathExists(const CanonPath & path) override
+ {
+ checkAccess(path);
+ return next->pathExists(prefix + path);
+ }
+
+ Stat lstat(const CanonPath & path) override
+ {
+ checkAccess(path);
+ return next->lstat(prefix + path);
+ }
+
+ DirEntries readDirectory(const CanonPath & path) override
+ {
+ checkAccess(path);
+ DirEntries entries;
+ for (auto & entry : next->readDirectory(prefix + path)) {
+ if (isAllowed(path + entry.first))
+ entries.insert(std::move(entry));
+ }
+ return entries;
+ }
+
+ std::string readLink(const CanonPath & path) override
+ {
+ checkAccess(path);
+ return next->readLink(prefix + path);
+ }
+
+ void checkAccess(const CanonPath & path)
+ {
+ if (!isAllowed(path))
+ throw Error("access to path '%s' has been filtered out", showPath(path));
+ }
+
+ bool isAllowed(const CanonPath & path)
+ {
+ auto i = cache.find(path);
+ if (i != cache.end()) return i->second;
+ auto res = isAllowedUncached(path);
+ cache.emplace(path, res);
+ return res;
+ }
+
+ bool isAllowedUncached(const CanonPath & path)
+ {
+ if (!path.isRoot() && !isAllowed(*path.parent())) return false;
+ // Note that unlike 'builtins.{path,filterSource}', we don't
+ // pass the prefix to the filter function.
+ return state.callPathFilter(filterFun, {next, prefix + path}, path.abs(), pos);
+ }
+
+ std::string showPath(const CanonPath & path) override
+ {
+ return next->showPath(prefix + path);
+ }
+};
+
+static void prim_filterPath(EvalState & state, PosIdx pos, Value * * args, Value & v)
+{
+ std::optional<SourcePath> path;
+ Value * filterFun = nullptr;
+ PathSet context;
+
+ state.forceAttrs(*args[0], pos);
+
+ for (auto & attr : *args[0]->attrs) {
+ auto n = state.symbols[attr.name];
+ if (n == "path")
+ path.emplace(state.coerceToPath(attr.pos, *attr.value, context));
+ else if (n == "filter") {
+ state.forceValue(*attr.value, pos);
+ filterFun = attr.value;
+ }
+ else
+ state.debugThrowLastTrace(EvalError({
+ .msg = hintfmt("unsupported argument '%1%' to 'filterPath'", state.symbols[attr.name]),
+ .errPos = state.positions[attr.pos]
+ }));
+ }
+
+ if (!path)
+ state.debugThrowLastTrace(EvalError({
+ .msg = hintfmt("'path' required"),
+ .errPos = state.positions[pos]
+ }));
+
+ if (!filterFun)
+ state.debugThrowLastTrace(EvalError({
+ .msg = hintfmt("'filter' required"),
+ .errPos = state.positions[pos]
+ }));
+
+ if (!context.empty())
+ state.debugThrowLastTrace(EvalError({
+ .msg = hintfmt("'path' argument to 'filterPath' cannot have a context"),
+ .errPos = state.positions[pos]
+ }));
+
+ auto accessor = make_ref<FilteringInputAccessor>(state, pos, *path, filterFun);
+
+ state.registerAccessor(accessor);
+
+ v.mkPath(accessor->root());
+}
+
+static RegisterPrimOp primop_filterPath({
+ .name = "__filterPath",
+ .args = {"args"},
+ .doc = R"(
+ This function lets you filter out files from a path. It takes a
+ path and a predicate function, and returns a new path from which
+ every file has been removed for which the predicate function
+ returns `false`.
+
+ For example, the following filters out all regular files in
+ `./doc` that don't end with the extension `.md`:
+
+ ```nix
+ builtins.filterPath {
+ path = ./doc;
+ filter =
+ path: type:
+ (type != "regular" || hasSuffix ".md" path);
+ }
+ ```
+
+ The filter function is called for all files in `path`. It takes
+ two arguments. The first is a string that represents the path of
+ the file to be filtered, relative to `path` (i.e. it does *not*
+ contain `./doc` in the example above). The second is the file
+ type, which can be one of `regular`, `directory` or `symlink`.
+
+ Note that unlike `builtins.filterSource` and `builtins.path`,
+ this function does not copy the result to the Nix store. Rather,
+ the result is a virtual path that lazily applies the filter
+ predicate. The result will only be copied to the Nix store if
+ needed (e.g. if used in a derivation attribute like `src =
+ builtins.filterPath { ... }`).
+ )",
+ .fun = prim_filterPath,
+ .experimentalFeature = Xp::Flakes,
+});
+
+}

tests/flakes/tree-operators.sh

@@ -0,0 +1,48 @@
+source ./common.sh
+
+flake1Dir=$TEST_ROOT/flake1
+
+mkdir -p $flake1Dir
+
+pwd=$(pwd)
+
+cat > $flake1Dir/flake.nix <<EOF
+{
+ inputs.docs = {
+ url = "path:https://$pwd/../../doc";
+ flake = false;
+ };
+
+ outputs = { self, docs }: with import ./lib.nix; {
+
+ mdOnly = builtins.filterPath {
+ path = docs;
+ filter =
+ path: type:
+ (type != "regular" || hasSuffix ".md" path);
+ };
+
+ noReleaseNotes = builtins.filterPath {
+ path = self.mdOnly + "/manual";
+ filter =
+ path: type:
+ assert !hasPrefix "/manual" path;
+ (builtins.baseNameOf path != "release-notes");
+ };
+
+ };
+}
+EOF
+
+cp ../lang/lib.nix $flake1Dir/
+
+nix build --out-link $TEST_ROOT/result $flake1Dir#mdOnly
+[[ -e $TEST_ROOT/result/manual/src/quick-start.md ]]
+[[ -e $TEST_ROOT/result/manual/src/release-notes ]]
+(! find $TEST_ROOT/result/ -type f | grep -v '.md$')
+find $TEST_ROOT/result/ -type f | grep release-notes
+
+nix build --out-link $TEST_ROOT/result $flake1Dir#noReleaseNotes
+[[ -e $TEST_ROOT/result/src/quick-start.md ]]
+(! [[ -e $TEST_ROOT/result/src/release-notes ]])
+(! find $TEST_ROOT/result/ -type f | grep release-notes)

tests/lang/lib.nix

@@ -19,6 +19,8 @@ rec {
 
  sum = foldl' (x: y: add x y) 0;
 
+ hasPrefix = pref: str: substring 0 (stringLength pref) str == pref;
+
  hasSuffix = ext: fileName:
  let lenFileName = stringLength fileName;
  lenExt = stringLength ext;

tests/local.mk

@@ -10,6 +10,7 @@ nix_tests = \
  flakes/unlocked-override.sh \
  flakes/absolute-paths.sh \
  flakes/build-paths.sh \
+ flakes/tree-operators.sh \
  ca/gc.sh \
  gc.sh \
  remote-store.sh \