diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000..9c0c0946adc --- /dev/null +++ b/.clang-format @@ -0,0 +1,30 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +BreakBeforeBraces: Custom +BraceWrapping: + AfterStruct: true + AfterClass: true + AfterFunction: true + AfterUnion: true + SplitEmptyRecord: false +PointerAlignment: Middle +FixNamespaceComments: false +SortIncludes: Never +#IndentPPDirectives: BeforeHash +SpaceAfterCStyleCast: true +SpaceAfterTemplateKeyword: false +AccessModifierOffset: -4 +AlignAfterOpenBracket: AlwaysBreak +AlignEscapedNewlines: DontAlign +ColumnLimit: 120 +BreakStringLiterals: false +BitFieldColonSpacing: None +AllowShortFunctionsOnASingleLine: Empty +AlwaysBreakTemplateDeclarations: Yes +BinPackParameters: false +BreakConstructorInitializers: BeforeComma +EmptyLineAfterAccessModifier: Leave # change to always/never later? +EmptyLineBeforeAccessModifier: Leave +#PackConstructorInitializers: BinPack +BreakBeforeBinaryOperators: NonAssignment +AlwaysBreakBeforeMultilineStrings: true diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000000..0887b867087 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,3 @@ +# We use pointers to aggregates in a couple of places, intentionally. +# void * would look weird. +Checks: '-bugprone-sizeof-expression' diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 39d595199d8..526fecabf29 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,16 +10,8 @@ # This file .github/CODEOWNERS @edolstra -# Public documentation -/doc @fricklerhandwerk -*.md @fricklerhandwerk - # Documentation of built-in functions -src/libexpr/primops.cc @fricklerhandwerk @roberth -# Documentation on experimental features -src/libutil/experimental-features.cc @fricklerhandwerk -# Documentation on configuration settings -src/libstore/globals.hh @fricklerhandwerk +src/libexpr/primops.cc @roberth # Libstore layer /src/libstore @thufschmitt diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 217b19108cf..d12a4d36c5c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,6 +10,8 @@ -# Priorities +# Priorities and Process Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). + +The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 85ddcfad3bb..5b75704b53f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.2.0 + uses: zeebe-io/backport-action@v2.4.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa2551424ba..fdd2d67f6ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,12 +20,12 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v13 + - uses: cachix/cachix-action@v14 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -62,10 +62,10 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - uses: cachix/cachix-action@v13 + - uses: cachix/cachix-action@v14 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,12 +114,12 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v13 + - uses: cachix/cachix-action@v14 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' diff --git a/.gitignore b/.gitignore index d9f9d949b66..a0a0786ede9 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,7 @@ perl/Makefile.config /tests/functional/ca/config.nix /tests/functional/dyn-drv/config.nix /tests/functional/repl-result-out +/tests/functional/debugger-test-out /tests/functional/test-libstoreconsumer/test-libstoreconsumer # /tests/functional/lang/ @@ -141,6 +142,7 @@ compile_commands.json nix-rust/target result +result-* # IDE .vscode/ diff --git a/.version b/.version index 7329e21c3bd..db65e2167ef 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.20.0 +2.21.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ffcc0268f56..a0c2b16f496 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - Functional tests – [`tests/functional/**.sh`](./tests/functional) - Unit tests – [`src/*/tests`](./src/) - Integration tests – [`tests/nixos/*`](./tests/nixos) - - [ ] User documentation in the [manual](..doc/manual/src) + - [ ] User documentation in the [manual](./doc/manual/src) - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made diff --git a/Makefile b/Makefile index 41f14ac9215..d3542c3e981 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,12 @@ +# External build directory support + include mk/build-dir.mk -include $(buildprefix)Makefile.config clean-files += $(buildprefix)Makefile.config +# List makefiles + ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ @@ -24,7 +28,7 @@ makefiles = \ misc/upstart/local.mk endif -ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) +ifeq ($(ENABLE_UNIT_TESTS), yes) makefiles += \ tests/unit/libutil/local.mk \ tests/unit/libutil-support/local.mk \ @@ -34,18 +38,28 @@ makefiles += \ tests/unit/libexpr-support/local.mk endif -ifeq ($(ENABLE_TESTS), yes) +ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) makefiles += \ tests/functional/local.mk \ tests/functional/ca/local.mk \ tests/functional/dyn-drv/local.mk \ tests/functional/test-libstoreconsumer/local.mk \ tests/functional/plugins/local.mk -else -makefiles += \ - mk/disable-tests.mk endif +# Some makefiles require access to built programs and must be included late. +makefiles-late = + +ifeq ($(ENABLE_DOC_GEN), yes) +makefiles-late += doc/manual/local.mk +endif + +ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) +makefiles-late += doc/internal-api/local.mk +endif + +# Miscellaneous global Flags + OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) @@ -55,13 +69,55 @@ else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE endif -include mk/lib.mk +include mk/platform.mk -# Must be included after `mk/lib.mk` so rules refer to variables defined -# by the library. Rules are not "lazy" like variables, unfortunately. -ifeq ($(ENABLE_BUILD), yes) -$(eval $(call include-sub-makefile, doc/manual/local.mk)) -$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +ifdef HOST_WINDOWS + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src + +# Include the main lib, causing rules to be defined + +include mk/lib.mk + +# Fallback stub rules for better UX when things are disabled +# +# These must be defined after `mk/lib.mk`. Otherwise the first rule +# incorrectly becomes the default target. + +ifneq ($(ENABLE_UNIT_TESTS), yes) +.PHONY: check +check: + @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." + @exit 1 +endif + +ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) +.PHONY: installcheck +installcheck: + @echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'." + @exit 1 +endif + +# Documentation fallback stub rules. + +ifneq ($(ENABLE_DOC_GEN), yes) +.PHONY: manual-html manpages +manual-html manpages: + @echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'." + @exit 1 +endif + +ifneq ($(ENABLE_INTERNAL_API_DOCS), yes) +.PHONY: internal-api-html +internal-api-html: + @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." + @exit 1 +endif diff --git a/Makefile.config.in b/Makefile.config.in index c85e028c2eb..d5c382630f2 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -9,8 +9,11 @@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_BUILD = @ENABLE_BUILD@ +ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ +ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ +ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@ ENABLE_S3 = @ENABLE_S3@ -ENABLE_TESTS = @ENABLE_TESTS@ +ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ @@ -26,7 +29,6 @@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ -RAPIDCHECK_HEADERS = @RAPIDCHECK_HEADERS@ SHELL = @bash@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ @@ -36,12 +38,10 @@ checkbindir = @checkbindir@ checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ -doc_generate = @doc_generate@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ -internal_api_docs = @internal_api_docs@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ diff --git a/configure.ac b/configure.ac index f8b937eb568..676b145a563 100644 --- a/configure.ac +++ b/configure.ac @@ -47,6 +47,10 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var +# Assign a default value to C{,XX}FLAGS as the default configure script sets them +# to -O2 otherwise, which we don't want to have hardcoded +CFLAGS=${CFLAGS-""} +CXXFLAGS=${CXXFLAGS-""} AC_PROG_CC AC_PROG_CXX @@ -122,7 +126,6 @@ AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) @@ -133,6 +136,48 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s AC_SUBST(storedir) +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) + +# Building without unit tests is useful for bootstrapping with a smaller footprint +# or running the tests in a separate derivation. Otherwise, we do compile and +# run them. + +AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), + ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) +AC_SUBST(ENABLE_UNIT_TESTS) + +AS_IF( + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], + [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) + +AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), + ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) +AC_SUBST(ENABLE_FUNCTIONAL_TESTS) + +# documentation generation switch +AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), + ENABLE_DOC_GEN=$enableval, ENABLE_DOC_GEN=$ENABLE_BUILD) +AC_SUBST(ENABLE_DOC_GEN) + +AS_IF( + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"], + [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])]) + +# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. +AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), + ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no) +AC_SUBST(ENABLE_INTERNAL_API_DOCS) + +AS_IF( + [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], + [NEED_PROG(jq, jq)]) + +AS_IF([test "$ENABLE_BUILD" == "yes"],[ + # Look for boost, a required dependency. # Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, # and CPPFLAGS are not passed to the C++ compiler automatically. @@ -155,18 +200,6 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi -# Running the functional tests without building Nix is useful for testing -# different pre-built versions of Nix against each other. -AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), - ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) -AC_SUBST(ENABLE_BUILD) -# Building without tests is useful for bootstrapping with a smaller footprint -# or running the tests in a separate derivation. Otherwise, we do compile and -# run them. -AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) -AC_SUBST(ENABLE_TESTS) - AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) AC_SUBST(INSTALL_UNIT_TESTS) @@ -179,11 +212,6 @@ AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to i checklibdir=$withval, checklibdir=$libdir) AC_SUBST(checklibdir) -# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. -AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), - internal_api_docs=$enableval, internal_api_docs=no) -AC_SUBST(internal_api_docs) - # LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]), lto=$enableval, lto=no) @@ -227,17 +255,25 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) -# Look for editline, a required dependency. +# Look for editline or readline, a required dependency. # The the libeditline.pc file was added only in libeditline >= 1.15.2, # see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, -# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for -# editline.h when the pkg-config approach fails. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ - AC_CHECK_HEADERS([editline.h], [true], - [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) - AC_SEARCH_LIBS([readline read_history], [editline], [], - [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) -]) +# Older versions are no longer supported. +AC_ARG_WITH( + [readline-flavor], + AS_HELP_STRING([--with-readline-flavor],[Which library to use for nice line editting with the Nix language REPL" [default=editline]]), + [readline_flavor=$withval], + [readline_flavor=editline]) +AS_CASE(["$readline_flavor"], + [editline], [ + readline_flavor_pc=libeditline + ], + [readline], [ + readline_flavor_pc=readline + AC_DEFINE([USE_READLINE], [1], [Use readline instead of editline]) + ], + [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])]) +PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) # Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) @@ -283,7 +319,13 @@ esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Optional dependencies for better normalizing file system data -AC_CHECK_HEADERS[sys/xattr.h] +AC_CHECK_HEADERS([sys/xattr.h]) +AS_IF([test "$ac_cv_header_sys_xattr_h" = "yes"],[ + AC_CHECK_FUNCS([llistxattr lremovexattr]) + AS_IF([test "$ac_cv_func_llistxattr" = "yes" && test "$ac_cv_func_lremovexattr" = "yes"],[ + AC_DEFINE([HAVE_ACL_SUPPORT], [1], [Define if we can manipulate file system Access Control Lists]) + ]) +]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) @@ -310,48 +352,35 @@ if test "$gc" = yes; then AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) fi - -if test "$ENABLE_TESTS" = yes; then +AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ # Look for gtest. -PKG_CHECK_MODULES([GTEST], [gtest_main]) - +PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main]) # Look for rapidcheck. -AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK]) -# No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 -AC_LANG_PUSH(C++) -AC_SUBST(RAPIDCHECK_HEADERS) -[CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] -[LIBS="-lrapidcheck -lgtest $LIBS"] -AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) -dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols -AC_LINK_IFELSE([ - AC_LANG_PROGRAM([[ - #include - #include - ]], [[ - return RUN_ALL_TESTS(); - ]]) - ], - [], - [AC_MSG_ERROR([librapidcheck is not found.])]) -AC_LANG_POP(C++) +PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) -fi +]) # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) -# documentation generation switch -AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), - doc_generate=$enableval, doc_generate=yes) -AC_SUBST(doc_generate) - - # Look for lowdown library. -PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) +AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]), + enable_markdown=$enableval, enable_markdown=auto) +AS_CASE(["$enable_markdown"], + [yes | auto], [ + PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [ + CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS" + have_lowdown=1 + AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.]) + ], [ + AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])]) + ]) + ], + [no], [have_lowdown=], + [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])]) # Look for libgit2. @@ -388,6 +417,8 @@ if test "$embedded_sandbox_shell" = yes; then AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) fi +]) + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff --git a/boehmgc-coroutine-sp-fallback.diff b/dep-patches/boehmgc-coroutine-sp-fallback.diff similarity index 100% rename from boehmgc-coroutine-sp-fallback.diff rename to dep-patches/boehmgc-coroutine-sp-fallback.diff diff --git a/boehmgc-traceable_allocator-public.diff b/dep-patches/boehmgc-traceable_allocator-public.diff similarity index 100% rename from boehmgc-traceable_allocator-public.diff rename to dep-patches/boehmgc-traceable_allocator-public.diff diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index ad5af97e6fe..6c6c325bdc9 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -81,7 +81,7 @@ EXPAND_ONLY_PREDEF = YES # RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. -INCLUDE_PATH = @RAPIDCHECK_HEADERS@ +INCLUDE_PATH = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The diff --git a/doc/internal-api/local.mk b/doc/internal-api/local.mk index 890f341b71e..bf2c4dedea1 100644 --- a/doc/internal-api/local.mk +++ b/doc/internal-api/local.mk @@ -1,19 +1,7 @@ -.PHONY: internal-api-html - -ifeq ($(internal_api_docs), yes) - $(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg mkdir -p $(docdir)/internal-api { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen - # Generate the HTML API docs for Nix's unstable internal interfaces. +.PHONY: internal-api-html internal-api-html: $(docdir)/internal-api/html/index.html - -else - -# Make a nicer error message -internal-api-html: - @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." - @exit 1 - -endif diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 73fb7e75e24..d524dbb1309 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -6,6 +6,8 @@ additional-css = ["custom.css"] additional-js = ["redirects.js"] edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" git-repository-url = "https://github.com/NixOS/nix" +fold.enable = true +fold.level = 1 [preprocessor.anchors] renderers = ["html"] diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index ae31b2a1f61..ba5667a4305 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -93,9 +93,6 @@ let maybeProse = # FIXME: this is a horrible hack to keep `nix help-stores` working. - # the correct answer to this is to remove that command and replace it - # by statically generated manpages or the output of something like `nix - # store info `. let help-stores = '' ${index} @@ -121,7 +118,7 @@ let }; in optionalString (details ? doc) ( - if match "@store-types@" details.doc != [ ] + if match ".*@store-types@.*" details.doc != null then help-stores else details.doc ); diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 456000d3d5b..b771688854a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,3 @@ -ifeq ($(doc_generate),yes) - # The version of Nix used to generate the doc. Can also be # `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), # if one prefers. @@ -180,6 +178,8 @@ manual-html: $(docdir)/manual/index.html install: $(docdir)/manual/index.html # Generate 'nix' manpages. +.PHONY: manpages +manpages: $(mandir)/man1/nix3-manpages install: $(mandir)/man1/nix3-manpages man: doc/manual/generated/man1/nix3-manpages all: doc/manual/generated/man1/nix3-manpages @@ -225,5 +225,3 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ @rm -rf $(DESTDIR)$(docdir)/manual @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual @rm -rf $(DESTDIR)$(docdir)/manual.tmp - -endif diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 3b507adf341..28b80f5891e 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -18,9 +18,10 @@ const redirects = { "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html", "chap-diff-hook": "advanced-topics/diff-hook.html", "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered", - "chap-distributed-builds": "advanced-topics/distributed-builds.html", + "chap-distributed-builds": "command-ref/conf-file.html#conf-builders", "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", + "chap-writing-nix-expressions": "language/index.html", "part-command-ref": "command-ref/command-ref.html", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", @@ -357,7 +358,11 @@ const redirects = { "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", "characterization-testing": "#characterisation-testing-unit", - } + }, + "glossary.html": { + "gloss-local-store": "store/types/local-store.html", + "gloss-chroot-store": "store/types/local-store.html", + }, }; // the following code matches the current page's URL against the set of redirects. diff --git a/doc/manual/rl-next/better-errors-in-nix-repl.md b/doc/manual/rl-next/better-errors-in-nix-repl.md new file mode 100644 index 00000000000..4deaa8c7041 --- /dev/null +++ b/doc/manual/rl-next/better-errors-in-nix-repl.md @@ -0,0 +1,40 @@ +--- +synopsis: Concise error printing in `nix repl` +prs: 9928 +--- + +Previously, if an element of a list or attribute set threw an error while +evaluating, `nix repl` would print the entire error (including source location +information) inline. This output was clumsy and difficult to parse: + +``` +nix-repl> { err = builtins.throw "uh oh!"; } +{ err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } +``` + +Now, only the error message is displayed, making the output much more readable. +``` +nix-repl> { err = builtins.throw "uh oh!"; } +{ err = «error: uh oh!»; } +``` + +However, if the whole expression being evaluated throws an error, source +locations and (if applicable) a stack trace are printed, just like you'd expect: + +``` +nix-repl> builtins.throw "uh oh!" +error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! +``` + diff --git a/doc/manual/rl-next/debugger-locals-for-let-expressions.md b/doc/manual/rl-next/debugger-locals-for-let-expressions.md new file mode 100644 index 00000000000..7362087244a --- /dev/null +++ b/doc/manual/rl-next/debugger-locals-for-let-expressions.md @@ -0,0 +1,9 @@ +--- +synopsis: "`--debugger` can now access bindings from `let` expressions" +prs: 9918 +issues: 8827. +--- + +Breakpoints and errors in the bindings of a `let` expression can now access +those bindings in the debugger. Previously, only the body of `let` expressions +could access those bindings. diff --git a/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md b/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md new file mode 100644 index 00000000000..c9322581608 --- /dev/null +++ b/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md @@ -0,0 +1,25 @@ +--- +synopsis: The `--debugger` will start more reliably in `let` expressions and function calls +prs: 9917 +issues: 6649 +--- + +Previously, if you attempted to evaluate this file with the debugger: + +```nix +let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); +in + b +``` + +Nix would correctly enter the debugger at `builtins.break a`, but if you asked +it to `:continue`, it would skip over the `builtins.break "hello"` expression +entirely. + +Now, Nix will correctly enter the debugger at both breakpoints. diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md deleted file mode 100644 index 73e6fbb24d8..00000000000 --- a/doc/manual/rl-next/hash-format-nix32.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -synopsis: Rename hash format `base32` to `nix32` -prs: 9452 ---- - -Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for -[Base32](https://en.wikipedia.org/wiki/Base32). - -## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` - -For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` -parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value -remains as a deprecated alias for `"base32"`. Please convert your code from: - -```nix -builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} -``` - -to - -```nix -builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} -``` \ No newline at end of file diff --git a/doc/manual/rl-next/leading-period.md b/doc/manual/rl-next/leading-period.md new file mode 100644 index 00000000000..ef7c2326f91 --- /dev/null +++ b/doc/manual/rl-next/leading-period.md @@ -0,0 +1,10 @@ +--- +synopsis: Store paths are allowed to start with `.` +issues: 912 +prs: 9867 9091 9095 9120 9121 9122 9130 9219 9224 +--- + +Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. +From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + +Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md deleted file mode 100644 index 6df44dbb62c..00000000000 --- a/doc/manual/rl-next/mounted-ssh-store.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: Mounted SSH Store -issues: 7890 -prs: 7912 ---- - -Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). -This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md deleted file mode 100644 index 26b961b7686..00000000000 --- a/doc/manual/rl-next/nix-config-show.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: Rename to `nix config show` -issues: 7672 -prs: 9477 ---- - -`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md deleted file mode 100644 index 734cefd1b3a..00000000000 --- a/doc/manual/rl-next/nix-env-json-drv-path.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: Fix `nix-env --query --drv-path --json` -prs: 9257 ---- - -Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md deleted file mode 100644 index 2b718a66b7e..00000000000 --- a/doc/manual/rl-next/nix-hash-convert.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -synopsis: Add `nix hash convert` -prs: 9452 ---- - -New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track -to stabilization! Examples: - -- Convert the hash to `nix32`. - - ```bash - $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df" - vw46m23bizj4n8afrc0fj19wrp7mj3c0 - ``` - `nix32` is a base32 encoding with a nix-specific character set. - Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input - hash. -- Convert the hash to the `sri` format that includes an algorithm specification: - ```bash - nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df" - sha1-gA1Zz808BekAy04hS+SPa4hqCN8= - ``` - or with an explicit `-to` format: - ```bash - nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df" - sha1-gA1Zz808BekAy04hS+SPa4hqCN8= - ``` -- Assert the input format of the hash: - ```bash - nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" - error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32' - nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" - sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= - ``` - -The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion. - -## Related Deprecations - -The following commands are still available but will emit a deprecation warning. Please convert your code to -`nix hash convert`: - -- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead. -- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead. -- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. -- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` - or even just `nix hash convert $hash1 $hash2` instead. diff --git a/doc/manual/rl-next/reduce-debugger-clutter.md b/doc/manual/rl-next/reduce-debugger-clutter.md new file mode 100644 index 00000000000..9bc902eee07 --- /dev/null +++ b/doc/manual/rl-next/reduce-debugger-clutter.md @@ -0,0 +1,37 @@ +--- +synopsis: "Visual clutter in `--debugger` is reduced" +prs: 9919 +--- + +Before: +``` +info: breakpoint reached + + +Starting REPL to allow you to inspect the current state of the evaluator. + +Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + +nix-repl> :continue +error: uh oh + + +Starting REPL to allow you to inspect the current state of the evaluator. + +Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + +nix-repl> +``` + +After: + +``` +info: breakpoint reached + +Nix 2.20.0pre20231222_dirty debugger +Type :? for help. +nix-repl> :continue +error: uh oh + +nix-repl> +``` diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md deleted file mode 100644 index 5b210289ddd..00000000000 --- a/doc/manual/rl-next/source-positions-in-errors.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -synopsis: Source locations are printed more consistently in errors -issues: 561 -prs: 9555 ---- - -Source location information is now included in error messages more -consistently. Given this code: - -```nix -let - attr = {foo = "bar";}; - key = {}; -in - attr.${key} -``` - -Previously, Nix would show this unhelpful message when attempting to evaluate -it: - -``` -error: - … while evaluating an attribute name - - error: value is a set while a string was expected -``` - -Now, the error message displays where the problematic value was found: - -``` -error: - … while evaluating an attribute name - - at bad.nix:4:11: - - 3| key = {}; - 4| in attr.${key} - | ^ - 5| - - error: value is a set while a string was expected -``` diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c67ddc6cbbe..70dea4fbd75 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -104,8 +104,12 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) -- [Protocols](protocols/index.md) +- [Formats and Protocols](protocols/index.md) + - [JSON Formats](protocols/json/index.md) + - [Store Object Info](protocols/json/store-object-info.md) + - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Store Path Specification](protocols/store-path.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/index.md) @@ -117,6 +121,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) diff --git a/doc/manual/_redirects b/doc/manual/src/_redirects similarity index 97% rename from doc/manual/_redirects rename to doc/manual/src/_redirects index 62c693c9737..8bf0e854b26 100644 --- a/doc/manual/_redirects +++ b/doc/manual/src/_redirects @@ -36,5 +36,6 @@ /package-management/s3-substituter /store/types/s3-binary-cache-store 301! /protocols/protocols /protocols 301! +/json/* /protocols/json/:splat 301! /release-notes/release-notes /release-notes 301! diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 507c5ecb769..52acd039c10 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -36,16 +36,8 @@ error: cannot connect to 'mac' then you need to ensure that the `PATH` of non-interactive login shells contains Nix. -> **Warning** -> -> If you are building via the Nix daemon, it is the Nix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine. -> -> If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine. - -The list of remote machines can be specified on the command line or in -the Nix configuration file. The former is convenient for testing. For -example, the following command allows you to build a derivation for -`x86_64-darwin` on a Linux machine: +The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file. +For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine: ```console $ uname @@ -60,97 +52,20 @@ $ cat ./result Darwin ``` -It is possible to specify multiple builders separated by a semicolon or -a newline, e.g. +It is possible to specify multiple build machines separated by a semicolon or a newline, e.g. ```console --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' ``` -Each machine specification consists of the following elements, separated -by spaces. Only the first element is required. To leave a field at its -default, set it to `-`. - -1. The URI of the remote store in the format - `ssh://[username@]hostname`, e.g. `ssh://nix@mac` or `ssh://mac`. - For backward compatibility, `ssh://` may be omitted. The hostname - may be an alias defined in your `~/.ssh/config`. - -2. A comma-separated list of Nix platform type identifiers, such as - `x86_64-darwin`. It is possible for a machine to support multiple - platform types, e.g., `i686-linux,x86_64-linux`. If omitted, this - defaults to the local platform type. - -3. The SSH identity file to be used to log in to the remote machine. If - omitted, SSH will use its regular identities. - -4. The maximum number of builds that Nix will execute in parallel on - the machine. Typically this should be equal to the number of CPU - cores. For instance, the machine `itchy` in the example will execute - up to 8 builds in parallel. - -5. The “speed factor”, indicating the relative speed of the machine. If - there are multiple machines of the right type, Nix will prefer the - fastest, taking load into account. - -6. A comma-separated list of *supported features*. If a derivation has - the `requiredSystemFeatures` attribute, then Nix will only perform - the derivation on a machine that has the specified features. For - instance, the attribute - - ```nix - requiredSystemFeatures = [ "kvm" ]; - ``` - - will cause the build to be performed on a machine that has the `kvm` - feature. - -7. A comma-separated list of *mandatory features*. A machine will only - be used to build a derivation if all of the machine’s mandatory - features appear in the derivation’s `requiredSystemFeatures` - attribute. - -8. The (base64-encoded) public host key of the remote machine. If omitted, SSH - will use its regular known-hosts file. Specifically, the field is calculated - via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`. - -For example, the machine specification - - nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm - nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 - nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark - -specifies several machines that can perform `i686-linux` builds. -However, `poochie` will only do builds that have the attribute - -```nix -requiredSystemFeatures = [ "benchmark" ]; -``` - -or - -```nix -requiredSystemFeatures = [ "benchmark" "kvm" ]; -``` - -`itchy` cannot do builds that require `kvm`, but `scratchy` does support -such builds. For regular builds, `itchy` will be preferred over -`scratchy` because it has a higher speed factor. - -Remote builders can also be configured in `nix.conf`, e.g. +Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g. builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd -Finally, remote builders can be configured in a separate configuration -file included in `builders` via the syntax `@file`. For example, +Finally, remote build machines can be configured in a separate configuration +file included in `builders` via the syntax `@/path/to/file`. For example, builders = @/etc/nix/machines -causes the list of machines in `/etc/nix/machines` to be included. (This -is the default.) - -If you want the builders to use caches, you likely want to set the -option `builders-use-substitutes` in your local `nix.conf`. - -To build only on remote builders and disable building on the local -machine, you can use the option `--max-jobs 0`. +causes the list of machines in `/etc/nix/machines` to be included. +(This is the default.) diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 3cab79f0eba..1bc88d85829 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -51,7 +51,7 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto - [`--delete-old`](#opt-delete-old) / `-d`\ Delete all old generations of profiles. - This is the equivalent of invoking `nix-env --delete-generations old` on each found profile. + This is the equivalent of invoking [`nix-env --delete-generations old`](@docroot@/command-ref/nix-env/delete-generations.md#generations-old) on each found profile. - [`--delete-older-than`](#opt-delete-older-than) *period*\ Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index adc6fc219dd..6b6ea798ead 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -12,13 +12,13 @@ This operation deletes the specified generations of the current profile. *generations* can be a one of the following: -- `...`:\ +- [`...`](#generations-list):\ A list of generation numbers, each one a separate command-line argument. Delete exactly the profile generations given by their generation number. Deleting the current generation is not allowed. -- The special value `old` +- [The special value `old`](#generations-old) Delete all generations except the current one. @@ -30,7 +30,7 @@ This operation deletes the specified generations of the current profile. > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. > They will also be deleted. -- `d`:\ +- [`d`](#generations-time):\ The last *number* days *Example*: `30d` @@ -38,7 +38,7 @@ This operation deletes the specified generations of the current profile. Delete all generations created more than *number* days ago, except the most recent one of them. This allows rolling back to generations that were available within the specified period. -- `+`:\ +- [`+`](#generations-count):\ The last *number* generations up to the present *Example*: `+5` diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index e1b4a3e8014..479c9abcf27 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -35,13 +35,51 @@ standard input. - `--parse`\ Just parse the input files, and print their abstract syntax trees on - standard output in ATerm format. + standard output as a Nix expression. - `--eval`\ Just parse and evaluate the input files, and print the resulting values on standard output. No instantiation of store derivations takes place. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` + - `--find-file`\ Look up the given files in Nix’s search path (as specified by the `NIX_PATH` environment variable). If found, print the corresponding @@ -61,11 +99,11 @@ standard input. - `--json`\ When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as an ATerm. + representation of the abstract syntax tree rather than as a Nix expression. - `--xml`\ When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as an ATerm. + representation of the abstract syntax tree rather than as a Nix expression. The schema is the same as that used by the [`toXML` built-in](../language/builtins.md). @@ -133,28 +171,24 @@ $ nix-instantiate --eval --xml --expr '1 + 2' The difference between non-strict and strict evaluation: ```console -$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }' -... - - - - - - -... -``` - -Note that `y` is left unevaluated (the XML representation doesn’t -attempt to show non-normal forms). +$ nix-instantiate --eval --xml --expr '{ x = {}; }' + + + + + + + + -```console -$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }' -... - - - - - - -... +$ nix-instantiate --eval --xml --strict --expr '{ x = {}; }' + + + + + + + + + ``` diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index 75226cd1a0c..1dddb207c3a 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -172,7 +172,7 @@ Please observe these guidelines to ease reviews: > ``` ```` - Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + Highlight syntax definitions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: ```` > **Syntax** diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 4d3d66397dd..916ec307716 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -31,7 +31,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix develop .#native-clang11StdenvPackages +$ nix develop .#native-clangStdenvPackages ``` > **Note** @@ -44,18 +44,21 @@ To build Nix itself in this shell: ```console [nix-shell]$ autoreconfPhase [nix-shell]$ configurePhase -[nix-shell]$ make -j $NIX_BUILD_CORES +[nix-shell]$ make -j $NIX_BUILD_CORES OPTIMIZE=0 ``` To install it in `$(pwd)/outputs` and test it: ```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ make install OPTIMIZE=0 +[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES [nix-shell]$ nix --version nix (Nix) 2.12 ``` +For more information on running and filtering tests, see +[`testing.md`](./testing.md). + To build a release version of Nix for the current operating system and CPU architecture: ```console @@ -75,7 +78,7 @@ $ nix-shell To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages ``` > **Note** @@ -108,6 +111,26 @@ $ nix-build You can also build Nix for one of the [supported platforms](#platforms). +## Makefile variables + +You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. + +Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables: + +- `ENABLE_BUILD=yes` to enable building the C++ code. +- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.). + + The docs can take a while to build, so you may want to disable this for local development. +- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. +- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. +- `OPTIMIZE=1` to enable optimizations. +- `libraries=libutil programs=` to only build a specific library. + + This will fail in the linking phase if the other libraries haven't been built, but is useful for checking types. +- `libraries= programs=nix` to only build a specific program. + + This will not work in general, because the programs need the libraries. + ## Platforms Nix can be built for various platforms, as specified in [`flake.nix`]: @@ -124,10 +147,10 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that -platform. Common solutions include [remote builders] and [binary format emulation] +platform. Common solutions include [remote build machines] and [binary format emulation] (only supported on NixOS). -[remote builders]: ../advanced-topics/distributed-builds.md +[remote builders]: @docroot@/language/derivations.md#attr-builder [binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems Given such a setup, executing the build only requires selecting the respective attribute. @@ -281,7 +304,6 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master ### Build process Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. -Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. ## Branches diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index d8d16237981..31c39c16c09 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -77,7 +77,7 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. +Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. ### Characterisation testing { #characaterisation-testing-unit } diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 07891175a4e..a71b2e2b31a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -3,10 +3,10 @@ - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a - store object. Derivations are typically specified in Nix expressions + store object. Derivations declared in Nix expressions are specified using the [`derivation` primitive](./language/derivations.md). These are translated into low-level *store derivations* (implicitly by - `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). + `nix-build`, or explicitly by `nix-instantiate`). [derivation]: #gloss-derivation @@ -14,6 +14,7 @@ A [derivation] represented as a `.drv` file in the [store]. It has a [store path], like any [store object]. + It is the [instantiated][instantiate] form of a derivation. Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` @@ -23,9 +24,9 @@ - [instantiate]{#gloss-instantiate}, instantiation - Translate a [derivation] into a [store derivation]. + Save an evaluated [derivation] as a [store derivation] in the Nix [store]. - See [`nix-instantiate`](./command-ref/nix-instantiate.md). + See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. [instantiate]: #gloss-instantiate @@ -36,7 +37,7 @@ This can be achieved by: - Fetching a pre-built [store object] from a [substituter] - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] - - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. @@ -58,23 +59,12 @@ - [store]{#gloss-store} - A collection of store objects, with operations to manipulate that collection. - See [Nix store](./store/index.md) for details. + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. - There are many types of stores. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - - From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. - Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. - Local stores can be used for building [derivations](#derivation). - See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. + There are many types of stores, see [Store Types](./store/types/index.md) for details. [store]: #gloss-store - [local store]: #gloss-local-store - -- [chroot store]{#gloss-chroot-store} - - A [local store] whose canonical path is anything other than `/nix/store`. - [binary cache]{#gloss-binary-cache} @@ -86,7 +76,7 @@ - [store path]{#gloss-store-path} - The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. > **Example** > @@ -126,7 +116,7 @@ non-[fixed-output](#gloss-fixed-output-derivation) derivation. -- [output-addressed store object]{#gloss-output-addressed-store-object} +- [content-addressed store object]{#gloss-content-addressed-store-object} A [store object] whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). @@ -155,6 +145,11 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. +- [impure derivation]{#gloss-impure-derivation} + + [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + - [Nix database]{#gloss-nix-database} An SQlite database to track [reference]s between [store object]s. @@ -166,11 +161,13 @@ - [Nix expression]{#gloss-nix-expression} - A high-level description of software packages and compositions - thereof. Deploying software using Nix entails writing Nix - expressions for your packages. Nix expressions are translated to - derivations that are stored in the Nix store. These derivations can - then be built. + 1. Commonly, a high-level description of software packages and compositions + thereof. Deploying software using Nix entails writing Nix + expressions for your packages. Nix expressions specify [derivations][derivation], + which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation]. + These derivations can then be [realised][realise] to produce [outputs][output]. + + 2. A syntactically valid use of the [Nix language]. For example, the contents of a `.nix` file form an expression. - [reference]{#gloss-reference} @@ -222,6 +219,9 @@ The [store derivation] that produced an [output path]. + The deriver for an output path can be queried with the `--deriver` option to + [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). + - [validity]{#gloss-validity} A store path is valid if all [store object]s in its [closure] can be read from the [store]. @@ -232,6 +232,7 @@ - All paths in the store path's [closure] are valid. [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md - [user environment]{#gloss-user-env} @@ -266,6 +267,21 @@ The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. +- [package]{#package} + + 1. A software package; a collection of files and other data. + + 2. A [package attribute set]. + +- [package attribute set]{#package-attribute-set} + + An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), + - attributes that declare something about how the package is supposed to be installed or used, + - other metadata or arbitrary attributes. + + [package attribute set]: #package-attribute-set + - [string interpolation]{#gloss-string-interpolation} Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. @@ -282,3 +298,6 @@ These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + + +[Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index ffabb250a84..0dc9891598a 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,89 +1,111 @@ # Installing a Binary Distribution -The easiest way to install Nix is to run the following command: +To install the latest version Nix, run the following command: ```console $ curl -L https://nixos.org/nix/install | sh ``` -This will run the installer interactively (causing it to explain what -it is doing more explicitly), and perform the default "type" of install -for your platform: -- single-user on Linux -- multi-user on macOS +This performs the default type of installation for your platform: - > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > - > - It took some time to support this cleanly. You may see posts, - > examples, and tutorials using obsolete workarounds. - > - Supporting it cleanly made macOS installs too complex to qualify - > as single-user, so this type is no longer supported on macOS. +- [Multi-user](#multi-user-installation): + - Linux with systemd and without SELinux + - macOS +- [Single-user](#single-user-installation): + - Linux without systemd + - Linux with SELinux -We recommend the multi-user install if it supports your platform and -you can authenticate with `sudo`. +We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`. -# Single User Installation - -To explicitly select a single-user installation on your system: +The installer can configured with various command line arguments and environment variables. +To show available command line flags: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ curl -L https://nixos.org/nix/install | sh -s -- --help ``` -This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You can run this under your usual user -account or root. The script will invoke `sudo` to create `/nix` -if it doesn’t already exist. If you don’t have `sudo`, you should -manually create `/nix` first as root, e.g.: +To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball). + +# Installing a pinned Nix version from a URL + +Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). +The directory for each version contains the corresponding SHA-256 hash. + +All installation scripts are invoked the same way: ```console -$ mkdir /nix -$ chown alice /nix +$ export VERSION=2.19.2 +$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh ``` -The install script will modify the first writable file from amongst -`.bash_profile`, `.bash_login` and `.profile` to source -`~/.nix-profile/etc/profile.d/nix.sh`. You can set the -`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing -the install script to disable this behaviour. - # Multi User Installation -The multi-user Nix installation creates system users, and a system -service for the Nix daemon. +The multi-user Nix installation creates system users and a system service for the Nix daemon. + +Supported systems: -**Supported Systems** - Linux running systemd, with SELinux disabled - macOS -You can instruct the installer to perform a multi-user installation on -your system: +To explicitly instruct the installer to perform a multi-user installation on your system: ```console $ curl -L https://nixos.org/nix/install | sh -s -- --daemon ``` -The multi-user installation of Nix will create build users between the -user IDs 30001 and 30032, and a group with the group ID 30000. You -can run this under your usual user account or root. The script -will invoke `sudo` as needed. +You can run this under your usual user account or `root`. +The script will invoke `sudo` as needed. + +# Single User Installation + +To explicitly select a single-user installation on your system: + +```console +$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +``` + +In a single-user installation, `/nix` is owned by the invoking user. +The script will invoke `sudo` to create `/nix` if it doesn’t already exist. +If you don’t have `sudo`, manually create `/nix` as `root`: + +```console +$ su root +# mkdir /nix +# chown alice /nix +``` + +# Installing from a binary tarball + +You can also download a binary tarball that contains Nix and all its dependencies: +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Download and unpack the tarball +- Run the installer -> **Note** +> **Example** > -> If you need Nix to use a different group ID or user ID set, you will -> have to download the tarball manually and [edit the install -> script](#installing-from-a-binary-tarball). +> ```console +> $ pushd $(mktemp -d) +> $ export VERSION=2.19.2 +> $ export SYSTEM=x86_64-linux +> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz +> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz +> $ cd nix-$VERSION-$SYSTEM +> $ ./install +> $ popd +> ``` -The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist. -The installer will first back up these files with a `.backup-before-nix` -extension. The installer will also create `/etc/profile.d/nix.sh`. +The installer can be customised with the environment variables declared in the file named `install-multi-user`. + +## Native packages for Linux distributions + +The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/). # macOS Installation + []{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} - -We believe we have ironed out how to cleanly support the read-only root +We believe we have ironed out how to cleanly support the read-only root file system on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, @@ -126,33 +148,3 @@ this to run the installer, but it may help if you run into trouble: boot process to avoid problems loading or restoring any programs that need access to your Nix store -# Installing a pinned Nix version from a URL - -Version-specific installation URLs for all Nix versions -since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). -The corresponding SHA-256 hash can be found in the directory for the given version. - -These install scripts can be used the same as usual: - -```console -$ curl -L https://releases.nixos.org/nix/nix-/install | sh -``` - -# Installing from a binary tarball - -You can also download a binary tarball that contains Nix and all its -dependencies. (This is what the install script at - does automatically.) You should unpack -it somewhere (e.g. in `/tmp`), and then run the script named `install` -inside the binary tarball: - -```console -$ cd /tmp -$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -$ cd nix-1.8-x86_64-darwin -$ ./install -``` - -If you need to edit the multi-user installation script to use different -group ID or a different user ID range, modify the variables set in the -file named `install-multi-user`. diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index d4babf1ea77..4aafa6d27ab 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -32,11 +32,15 @@ your distribution does not provide it, please install it from . - - The [Boehm garbage collector](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). To enable it, install + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce + the evaluator’s memory consumption (optional). + + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the flag `--enable-gc` to `configure`. + For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied. + - The `boost` library of version 1.66.0 or higher. It can be obtained from the official web site . @@ -72,7 +76,7 @@ This is an optional dependency and can be disabled by providing a `--disable-cpuid` to the `configure` script. - - Unless `./configure --disable-tests` is specified, GoogleTest (GTest) and + - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at and respectively. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 6d09f54d831..38edcdbc550 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -1,14 +1,40 @@ # Upgrading Nix -Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c -'nix-channel --update && -nix-env --install --attr nixpkgs.nix && -launchctl remove org.nixos.nix-daemon && -launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'` - -Single-user installations of Nix should run this: `nix-channel --update; -nix-env --install --attr nixpkgs.nix nixpkgs.cacert` - -Multi-user Nix users on Linux should run this with sudo: `nix-channel ---update; nix-env --install --attr nixpkgs.nix nixpkgs.cacert; systemctl -daemon-reload; systemctl restart nix-daemon` +> **Note** +> +> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md). + +Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`: + +```console +$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version" +nix (Nix) 2.18.1 +``` + +> **Warning** +> +> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema! +> Reverting to an older version of Nix may therefore require purging the store database before it can be used. + +## Linux multi-user + +```console +$ sudo su +# nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable +# systemctl daemon-reload +# systemctl restart nix-daemon +``` + +## macOS multi-user + +```console +$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo launchctl remove org.nixos.nix-daemon +$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist +``` + +## Single-user all platforms + +```console +$ nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable +``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 5a6c00cd4b6..7306fc182e0 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -257,7 +257,7 @@ Derivations can declare some infrequently used optional attributes. of the environment (typically, a few hundred kilobyte). - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\ - If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. + If this attribute is set to `true` and [distributed building is enabled](@docroot@/command-ref/conf-file.md#conf-builders), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. This is useful for derivations that are cheapest to build locally. - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\ diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 2aded5527b9..75f824a342b 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -36,7 +36,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect The system type on which the [`builder`](#attr-builder) executable is meant to be run. A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. - It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + It can automatically [build on other platforms](@docroot@/language/derivations.md#attr-builder) by forwarding build requests to other machines. [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system @@ -274,7 +274,7 @@ The [`builder`](#attr-builder) is executed as follows: directory (typically, `/nix/store`). - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the dervation. A detailed explanation of this + is set to `true` for the derivation. A detailed explanation of this behavior can be found in the [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 03b3f9d91b1..fb12ba51a8d 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -1,6 +1,8 @@ # Import From Derivation -The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). +The value of a Nix expression can depend on the contents of a [store object]. + +[store object]: @docroot@/glossary.md#gloss-store-object Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e9cbb5c9242..6fd66864bea 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -84,7 +84,7 @@ The `+` operator is overloaded to also work on strings and paths. > > *string* `+` *string* -Concatenate two [string]s and merge their string contexts. +Concatenate two [strings][string] and merge their string contexts. [String concatenation]: #string-concatenation @@ -94,7 +94,7 @@ Concatenate two [string]s and merge their string contexts. > > *path* `+` *path* -Concatenate two [path]s. +Concatenate two [paths][path]. The result is a path. [Path concatenation]: #path-concatenation @@ -150,9 +150,9 @@ If an attribute name is present in both, the attribute value from the latter is Comparison is -- [arithmetic] for [number]s -- lexicographic for [string]s and [path]s -- item-wise lexicographic for [list]s: +- [arithmetic] for [numbers][number] +- lexicographic for [strings][string] and [paths][path] +- item-wise lexicographic for [lists][list]: elements at the same index in both lists are compared according to their type and skipped if they are equal. All comparison operators are implemented in terms of `<`, and the following equivalencies hold: @@ -163,12 +163,12 @@ All comparison operators are implemented in terms of `<`, and the following equi | *a* `>` *b* | *b* `<` *a* | | *a* `>=` *b* | `! (` *a* `<` *b* `)` | -[Comparison]: #comparison-operators +[Comparison]: #comparison ## Equality -- [Attribute sets][attribute set] and [list]s are compared recursively, and therefore are fully evaluated. -- Comparison of [function]s always returns `false`. +- [Attribute sets][attribute set] and [lists][list] are compared recursively, and therefore are fully evaluated. +- Comparison of [functions][function] always returns `false`. - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index e999b287b9c..7d81c202095 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -20,6 +20,8 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write +[derivation]: ../glossary.md#gloss-derivation + ```nix "--with-freetype2-library=${freetype}/lib" ``` @@ -189,7 +191,7 @@ If neither is present, an error is thrown. > "${a}" > ``` > -> error: cannot coerce a set to a string +> error: cannot coerce a set to a string: { } > > at «string»:4:2: > diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index aea68a44170..74ffc70701b 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -156,6 +156,8 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + ## Attribute Set An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md new file mode 100644 index 00000000000..649d543ccf3 --- /dev/null +++ b/doc/manual/src/protocols/json/derivation.md @@ -0,0 +1,71 @@ +# Derivation JSON Format + +> **Warning** +> +> This JSON format is currently +> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> and subject to change. + +The JSON serialization of a +[derivations](@docroot@/glossary.md#gloss-store-derivation) +is a JSON object with the following fields: + +* `name`: + The name of the derivation. + This is used when calculating the store paths of the derivation's outputs. + +* `outputs`: + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: + + * `path`: The output path. + + * `hashAlgo`: + For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + + * `hash`: + For fixed-output derivations, the expected content hash in base-16. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + > "hashAlgo": "r:sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + +* `inputSrcs`: + A list of store paths on which this derivation depends. + +* `inputDrvs`: + A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations. + + > **Example** + > + > ```json + > "inputDrvs": { + > "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + + specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + +* `system`: + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: + The absolute path of the program to be executed to run the build. + Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: + The command-line arguments passed to the `builder`. + +* `env`: + The environment passed to the `builder`. diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md new file mode 100644 index 00000000000..ba4ab098f6a --- /dev/null +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -0,0 +1,98 @@ +# Store object info JSON format + +> **Warning** +> +> This JSON format is currently +> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> and subject to change. + +Info about a [store object]. + +* `path`: + + [Store path][store path] to the given store object. + +* `narHash`: + + Hash of the [file system object] part of the store object when serialized as a [Nix Archive]. + +* `narSize`: + + Size of the [file system object] part of the store object when serialized as a [Nix Archive]. + +* `references`: + + An array of [store paths][store path], possibly including this one. + +* `ca` (optional): + + Content address of this store object's file system object, used to compute its store path. + +[store path]: @docroot@/glossary.md#gloss-store-path +[file system object]: @docroot@/store/file-system-object.md +[Nix Archive]: @docroot@/glossary.md#gloss-nar + +## Impure fields + +These are not intrinsic properties of the store object. +In other words, the same store object residing in different store could have different values for these properties. + +* `deriver` (optional): + + The path to the [derivation] from which this store object is produced. + + [derivation]: @docroot@/glossary.md#gloss-store-derivation + +* `registrationTime` (optional): + + When this derivation was added to the store. + +* `ultimate` (optional): + + Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. + +* `signatures` (optional): + + Signatures claiming that this store object is what it claims to be. + Not relevant for [content-addressed] store objects, + but useful for [input-addressed] store objects. + + [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object + [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object + +### `.narinfo` extra fields + +This meta data is specific to the "binary cache" family of Nix store types. +This information is not intrinsic to the store object, but about how it is stored. + +* `url`: + + Where to download a compressed archive of the file system objects of this store object. + +* `compression`: + + The compression format that the archive is in. + +* `fileHash`: + + A digest for the compressed archive itself, as opposed to the data contained within. + +* `fileSize`: + + The size of the compressed archive itself. + +## Computed closure fields + +These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. + +* `closureSize`: + + The total size of the compressed archive itself for this object, and the compressed archive of every object in this object's [closure]. + +### `.narinfo` extra fields + +* `closureSize`: + + The total size of this store object and every other object in its [closure]. + +[closure]: @docroot@/glossary.md#gloss-closure diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md new file mode 100644 index 00000000000..fcf8038fcc3 --- /dev/null +++ b/doc/manual/src/protocols/store-path.md @@ -0,0 +1,126 @@ +# Complete Store Path Calculation + +This is the complete specification for how store paths are calculated. + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. + +Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +## Store path proper + +```ebnf +store-path = store-dir "/" digest "-" name +``` +where + +- `name` = the name of the store object. + +- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) + +- `digest` = base-32 representation of the first 160 bits of a [SHA-256] hash of `fingerprint` + + This the hash part of the store name + +## Fingerprint + +- ```ebnf + fingerprint = type ":" sha256 ":" inner-digest ":" store ":" name + ``` + + Note that it includes the location of the store as well as the name to make sure that changes to either of those are reflected in the hash + (e.g. you won't get `/nix/store/-name1` and `/nix/store/-name2`, or `/gnu/store/-name1`, with equal hash parts). + +- `type` = one of: + + - ```ebnf + | "text" ( ":" store-path )* + ``` + + for encoded derivations written to the store. + The optional trailing store paths are the references of the store object. + + - ```ebnf + | "source" ( ":" store-path )* + ``` + + For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. + Just like in the text case, we can have the store objects referenced by their paths. + Additionally, we can have an optional `:self` label to denote self reference. + + - ```ebnf + | "output:" id + ``` + + For either the outputs built from derivations, + paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. + (in that case "source" is used; this is only necessary for compatibility). + + `id` is the name of the output (usually, "out"). + For content-addressed store objects, `id`, is always "out". + +- `inner-digest` = base-16 representation of a SHA-256 hash of `inner-fingerprint` + +## Inner fingerprint + +- `inner-fingerprint` = one of the following based on `type`: + + - if `type` = `"text:" ...`: + + the string written to the resulting store path. + + - if `type` = `"source:" ...`: + + the the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. + + - if `type` = `"output:" id`: + + - For input-addressed derivation outputs: + + the [ATerm](@docroot@/protocols/derivation-aterm.md) serialization of the derivation modulo fixed output derivations. + + - For content-addressed store paths: + + ```ebnf + "fixed:out:" rec algo ":" hash ":" + ``` + + where + + - `rec` = one of: + + - ```ebnf + | "r:" + ``` + hashes of the for [Nix Archive (NAR)] (arbitrary file system object) serialization + + - ```ebnf + | "" + ``` + (empty string) for hashes of the flat (single file) serialization + + - ```ebnf + algo = "md5" | "sha1" | "sha256" + ``` + + - `hash` = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the path for fixed-output derivations). + + Note that `id` = `"out"`, regardless of the name part of the store path. + Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. + +[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR +[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 + +### Historical Note + +The `type` = `"source:" ...` and `type` = `"output:out"` grammars technically overlap in purpose, +in that both can represent data hashed by its SHA-256 NAR serialization. + +The original reason for this way of computing names was to prevent name collisions (for security). +For instance, the thinking was that it shouldn't be feasible to come up with a derivation whose output path collides with the path for a copied source. +The former would have an `inner-fingerprint` starting with `output:out:`, while the latter would have an `inner-fingerprint` starting with `source:`. + +Since `64519cfd657d024ae6e2bb74cb21ad21b886fd2a` (2008), however, it was decided that separating derivation-produced vs manually-hashed content-addressed data like this was not useful. +Now, data that is content-addressed with SHA-256 + NAR-serialization always uses the `source:...` construction, regardless of how it was produced (manually or by derivation). +This allows freely switching between using [fixed-output derivations](@docroot@/glossary.md#gloss-fixed-output-derivation) for fetching, and fetching out-of-band and then manually adding. +It also removes the ambiguity from the grammar. diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 04a0b7c96fa..75853ced787 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -10,7 +10,6 @@ For more in-depth information you are kindly referred to subsequent chapters. ``` The install script will use `sudo`, so make sure you have sufficient rights. - On Linux, `--daemon` can be omitted for a single-user install. For other installation methods, see the detailed [installation instructions](installation/index.md). diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md new file mode 100644 index 00000000000..26869e90a5f --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -0,0 +1,169 @@ +# Release 2.20.0 (2024-01-29) + +- Option `allowed-uris` can now match whole schemes in URIs without slashes [#9547](https://github.com/NixOS/nix/pull/9547) + + If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed. + Previously this only worked for schemes whose URIs used the `://` syntax. + +- Include cgroup stats when building through the daemon [#9598](https://github.com/NixOS/nix/pull/9598) + + Nix now also reports cgroup statistics when building through the Nix daemon and when doing remote builds using `ssh-ng`, + if both sides of the connection are using Nix 2.20 or newer. + +- Disallow empty search regex in `nix search` [#9481](https://github.com/NixOS/nix/pull/9481) + + [`nix search`](@docroot@/command-ref/new-cli/nix3-search.md) now requires a search regex to be passed. To show all packages, use `^`. + +- Add new `eval-system` setting [#4093](https://github.com/NixOS/nix/pull/4093) + + Add a new `eval-system` option. + Unlike `system`, it just overrides the value of `builtins.currentSystem`. + This is more useful than overriding `system`, because you can build these derivations on remote builders which can work on the given system. + In contrast, `system` also affects scheduling which will cause Nix to build those derivations locally even if that doesn't make sense. + + `eval-system` only takes effect if it is non-empty. + If empty (the default) `system` is used as before, so there is no breakage. + +- Import-from-derivation builds the derivation in the build store [#9661](https://github.com/NixOS/nix/pull/9661) + + When using `--eval-store`, `import`ing from a derivation will now result in the derivation being built on the build store, i.e. the store specified in the `store` Nix option. + + Because the resulting Nix expression must be copied back to the evaluation store in order to be imported, this requires the evaluation store to trust the build store's signatures. + +- Mounted SSH Store [#7890](https://github.com/NixOS/nix/issues/7890) [#7912](https://github.com/NixOS/nix/pull/7912) + + Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. + +- Rename `nix show-config` to `nix config show` [#7672](https://github.com/NixOS/nix/issues/7672) [#9477](https://github.com/NixOS/nix/pull/9477) + + `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command line interface. + +- Add command `nix hash convert` [#9452](https://github.com/NixOS/nix/pull/9452) + + This replaces the old `nix hash to-*` commands, which are still available but will emit a deprecation warning. Please convert as follows: + + - `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead. + - `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead. + - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. + - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` or even just `nix hash convert $hash1 $hash2` instead. + +- Rename hash format `base32` to `nix32` [#9452](https://github.com/NixOS/nix/pull/9452) + + Hash format `base32` was renamed to `nix32` since it used a special Nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + +- `nix profile` now allows referring to elements by human-readable names [#8678](https://github.com/NixOS/nix/pull/8678) + + [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Profile element names are generated when a package is installed and remain the same until the package is removed. + + **Warning**: The `manifest.nix` file used to record the contents of profiles has changed. Nix will automatically upgrade profiles to the new version when you modify the profile. After that, the profile can no longer be used by older versions of Nix. + +- Give `nix store add` a `--hash-algo` flag [#9809](https://github.com/NixOS/nix/pull/9809) + + Adds a missing feature that was present in the old CLI, and matches our + plans to have similar flags for `nix hash convert` and `nix hash path`. + +- Coercion errors include the failing value + + The `error: cannot coerce a to a string` message now includes the value + which caused the error. + + Before: + + ``` + error: cannot coerce a set to a string + ``` + + After: + + ``` + error: cannot coerce a set to a string: { aesSupport = «thunk»; + avx2Support = «thunk»; avx512Support = «thunk»; avxSupport = «thunk»; + canExecute = «thunk»; config = «thunk»; darwinArch = «thunk»; darwinMinVersion + = «thunk»; darwinMinVersionVariable = «thunk»; darwinPlatform = «thunk»; «84 + attributes elided»} + ``` + +- Type errors include the failing value + + In errors like `value is an integer while a list was expected`, the message now + includes the failing value. + + Before: + + ``` + error: value is a set while a string was expected + ``` + + After: + + ``` + error: expected a string but found a set: { ghc810 = «thunk»; + ghc8102Binary = «thunk»; ghc8107 = «thunk»; ghc8107Binary = «thunk»; + ghc865Binary = «thunk»; ghc90 = «thunk»; ghc902 = «thunk»; ghc92 = «thunk»; + ghc924Binary = «thunk»; ghc925 = «thunk»; «17 attributes elided»} + ``` + +- Source locations are printed more consistently in errors [#561](https://github.com/NixOS/nix/issues/561) [#9555](https://github.com/NixOS/nix/pull/9555) + + Source location information is now included in error messages more + consistently. Given this code: + + ```nix + let + attr = {foo = "bar";}; + key = {}; + in + attr.${key} + ``` + + Previously, Nix would show this unhelpful message when attempting to evaluate + it: + + ``` + error: + … while evaluating an attribute name + + error: value is a set while a string was expected + ``` + + Now, the error message displays where the problematic value was found: + + ``` + error: + … while evaluating an attribute name + + at bad.nix:4:11: + + 3| key = {}; + 4| in attr.${key} + | ^ + 5| + + error: expected a string but found a set + ``` + +- Some stack overflow segfaults are fixed [#9616](https://github.com/NixOS/nix/issues/9616) [#9617](https://github.com/NixOS/nix/pull/9617) + + The number of nested function calls has been restricted, to detect and report + infinite function call recursions. The default maximum call depth is 10,000 and + can be set with [the `max-call-depth` + option](@docroot@/command-ref/conf-file.md#conf-max-call-depth). + + This replaces the `stack overflow (possible infinite recursion)` message. + +- Better error reporting for `with` expressions [#9658](https://github.com/NixOS/nix/pull/9658) + + `with` expressions using non-attrset values to resolve variables are now reported with proper positions, e.g. + + ``` + nix-repl> with 1; a + error: + … while evaluating the first subexpression of a with expression + at «string»:1:1: + 1| with 1; a + | ^ + + error: expected a set but found an integer + ``` diff --git a/flake.lock b/flake.lock index 3cb9e72c974..f0efb4036ef 100644 --- a/flake.lock +++ b/flake.lock @@ -32,34 +32,18 @@ "type": "github" } }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1701355166, - "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=", + "lastModified": 1705033721, + "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c", + "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", "type": "github" }, "original": { "owner": "NixOS", - "ref": "staging-23.05", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } @@ -84,7 +68,6 @@ "inputs": { "flake-compat": "flake-compat", "libgit2": "libgit2", - "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" } diff --git a/flake.nix b/flake.nix index ada52c05d72..0bc70768e13 100644 --- a/flake.nix +++ b/flake.nix @@ -1,27 +1,24 @@ { description = "The purely functional package manager"; - # TODO Go back to nixos-23.05-small once - # https://github.com/NixOS/nixpkgs/pull/271202 is merged. - # - # Also, do not grab arbitrary further staging commits. This PR was - # carefully made to be based on release-23.05 and just contain - # rebuild-causing changes to packages that Nix actually uses. - # - # Once this is updated to something containing - # https://github.com/NixOS/nixpkgs/pull/271423, don't forget - # to remove the `nix.checkAllErrors = false;` line in the tests. - inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }: + outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; + officialRelease = false; # Set to true to build the release notes for the next release. @@ -53,7 +50,13 @@ "x86_64-w64-mingw32" ]; - stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; + stdenvs = [ + "ccacheStdenv" + "clangStdenv" + "gccStdenv" + "libcxxStdenv" + "stdenv" + ]; forAllSystems = lib.genAttrs systems; @@ -68,57 +71,6 @@ }) stdenvs); - # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 - # Not an "idiomatic" flake input because: - # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 - # - Subflake would download redundant and huge parent flake - # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 - inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) - fileset; - - baseFiles = - # .gitignore has already been processed, so any changes in it are irrelevant - # at this point. It is not represented verbatim for test purposes because - # that would interfere with repo semantics. - fileset.fileFilter (f: f.name != ".gitignore") ./.; - - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; - - nixSrc = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - ./boehmgc-coroutine-sp-fallback.diff - ./doc - ./misc - ./precompiled-headers.h - ./src - ./tests/unit - ./COPYING - ./scripts/local.mk - functionalTestFiles - ]); - }; - # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -143,407 +95,111 @@ cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - commonDeps = - { pkgs - , isStatic ? pkgs.stdenv.hostPlatform.isStatic - }: - with pkgs; rec { - # Use "busybox-sandbox-shell" if present, - # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ - "LDFLAGS=-fuse-ld=gold" - ]; - - testConfigureFlags = [ - "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" - ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]; - - internalApiDocsConfigureFlags = [ - "--enable-internal-api-docs" - ]; - - changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nativeBuildDeps = - [ - buildPackages.bison - buildPackages.flex - (lib.getBin buildPackages.lowdown-nix) - buildPackages.mdbook - buildPackages.mdbook-linkcheck - buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - - # Tests - buildPackages.git - buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq # Also for custom mdBook preprocessor. - buildPackages.openssh # only needed for tests (ssh-keygen) - ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] - # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d - ; - - buildDeps = - [ curl - bzip2 xz brotli - openssl sqlite - libarchive - (pkgs.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; - })) - boost - libsodium - ] - ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ - editline - lowdown-nix - ] - ++ lib.optional stdenv.isLinux libseccomp - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; - - checkDeps = [ - gtest - rapidcheck - ]; - - internalApiDocsDeps = [ - buildPackages.doxygen - ]; - - awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }); - - propagatedDeps = - [ ((boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }) - ) - nlohmann_json - ]; - }; - installScriptFor = tarballs: - with nixpkgsFor.x86_64-linux.native; - runCommand "installer-script" - { buildInputs = [ nix ]; - } - '' - mkdir -p $out/nix-support - - # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. - tarballPath() { - # Remove the store prefix - local path=''${1#${builtins.storeDir}/} - # Get the path relative to the derivation root - local rest=''${path#*/} - # Get the derivation hash - local drvHash=''${path%%-*} - echo "$drvHash/$rest" - } - - substitute ${./scripts/install.in} $out/install \ - ${pkgs.lib.concatMapStrings - (tarball: let - inherit (tarball.stdenv.hostPlatform) system; - in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ - '' - ) - tarballs - } --replace '@nixVersion@' ${version} - - echo "file installer $out/install" >> $out/nix-support/hydra-build-products - ''; - - testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { - NIX_DAEMON_PACKAGE = daemon; - NIX_CLIENT_PACKAGE = client; - name = - "nix-tests" - + optionalString - (versionAtLeast daemon.version "2.4pre20211005" && - versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - inherit version; - - src = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - functionalTestFiles - ]); + nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { + inherit tarballs; }; - VERSION_SUFFIX = versionSuffix; + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ./package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps ++ checkDeps; - propagatedBuildInputs = propagatedDeps; + inherit fileset; - enableParallelBuilding = true; + test-client = client; + test-daemon = daemon; - configureFlags = - testConfigureFlags # otherwise configure fails - ++ [ "--disable-build" ]; - dontBuild = true; - doInstallCheck = true; - - installPhase = '' - mkdir -p $out - ''; + doBuild = false; + }; - installCheckPhase = '' - mkdir -p src/nix-channel - make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES - ''; + binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { + inherit nix; }; - binaryTarball = nix: pkgs: + overlayFor = getStdenv: final: prev: let - inherit (pkgs) buildPackages; - inherit (pkgs) cacert; - installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; + stdenv = getStdenv final; in - - pkgs.runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${pkgs.system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --mtime='1970-01-01' \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''; - - overlayFor = getStdenv: final: prev: - let currentStdenv = getStdenv final; in { nixStable = prev.nix; - # Forward from the previous stage as we don’t want it to pick the lowdown override - nixUnstable = prev.nixUnstable; - - nix = - with final; - with commonDeps { - inherit pkgs; - inherit (currentStdenv.hostPlatform) isStatic; - }; - let - canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - in currentStdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; - inherit version; - - src = nixSrc; - VERSION_SUFFIX = versionSuffix; - - outputs = [ "out" "dev" "doc" ] - ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps - # There have been issues building these dependencies - ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps - ++ lib.optionals finalAttrs.doCheck checkDeps; - - propagatedBuildInputs = propagatedDeps; - - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString currentStdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ] ++ - lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ - [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ - lib.optionals finalAttrs.doCheck testConfigureFlags ++ - lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString currentStdenv.hostPlatform.isStatic '' - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString currentStdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - ''} + default-busybox-sandbox-shell = final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y ''; + }; - doInstallCheck = finalAttrs.doCheck; - installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - separateDebugInfo = !currentStdenv.hostPlatform.isStatic; - - strictDeps = true; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - - passthru.perl-bindings = final.callPackage ./perl { - inherit fileset; - stdenv = currentStdenv; - }; - - meta.platforms = lib.platforms.unix ++ lib.platforms.windows; - meta.mainProgram = "nix"; + libgit2-nix = final.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; }); - lowdown-nix = with final; currentStdenv.mkDerivation rec { - name = "lowdown-0.9.0"; + boehmgc-nix = (final.boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./dep-patches/boehmgc-coroutine-sp-fallback.diff - src = lowdown-src; + # https://github.com/ivmai/bdwgc/pull/586 + ./dep-patches/boehmgc-traceable_allocator-public.diff + ]; + }); - outputs = [ "out" "bin" "dev" ]; + changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - nativeBuildInputs = [ buildPackages.which ]; + nix = + let + officialRelease = false; + versionSuffix = + if officialRelease + then "" + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; + + in final.callPackage ./package.nix { + inherit + fileset + stdenv + versionSuffix + ; + officialRelease = false; + boehmgc = final.boehmgc-nix; + libgit2 = final.libgit2-nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + } // { + # this is a proper separate downstream package, but put + # here also for back compat reasons. + perl-bindings = final.nix-perl-bindings; + }; - configurePhase = '' - ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} - ./configure \ - PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "bin"}/bin - ''; + nix-perl-bindings = final.callPackage ./perl { + inherit fileset stdenv; }; + }; in { @@ -563,14 +219,25 @@ buildCross = forAllCrossSystems (crossSystem: lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];})); + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); buildNoTests = forAllSystems (system: - self.packages.${system}.nix.overrideAttrs (a: { - doCheck = - assert ! a?dontCheck; - false; - }) + self.packages.${system}.nix.override { + doCheck = false; + doInstallCheck = false; + installUnitTests = false; + } + ); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } ); # Perl bindings for various platforms. @@ -615,60 +282,17 @@ dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. - coverage = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; - - releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - src = nixSrc; - - configureFlags = testConfigureFlags; - - enableParallelBuilding = true; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps; - - dontInstall = false; - - doInstallCheck = true; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - hardeningDisable = ["fortify"]; - - NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; - }; + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; - - stdenv.mkDerivation { - pname = "nix-internal-api-docs"; - inherit version; - - src = nixSrc; - - configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - dontBuild = true; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; - }; + internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { + inherit fileset; + doBuild = false; + enableInternalAPIDocs = true; + }; # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { @@ -676,7 +300,9 @@ # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in runCommand "eval-nixos" { buildInputs = [ nix ]; } '' type -p nix-env @@ -729,14 +355,14 @@ rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; }); packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix; + inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; default = nix; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = nixpkgsFor.${system}.static.nix; @@ -768,47 +394,20 @@ stdenvs))); devShells = let - makeShell = pkgs: stdenv: - let - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - in - with commonDeps { inherit pkgs; }; - stdenv.mkDerivation { - name = "nix"; - - outputs = [ "out" "dev" "doc" ] - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; - - nativeBuildInputs = nativeBuildDeps - ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear - ++ lib.optional - (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) - pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if the current build doesn't need it - ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d - ; - - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags - ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - }; + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + ''; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + }); in forAllSystems (system: let diff --git a/maintainers/README.md b/maintainers/README.md index ee97c1195d6..fa321c7c09f 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -43,7 +43,11 @@ The team meets twice a week: - Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) - 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min) + 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). + Once a month, each team member checks the [Assigned](#assigned) column for prs/issues assigned to them, to either + - unblock it by providing input + - mark it as draft if it is blocked on the contributor + - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. - Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) diff --git a/maintainers/release-notes b/maintainers/release-notes index 34cd85a5692..2d84485c173 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,7 +1,5 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small -# ^^^^^^^ -# Only used for bash. shell.nix goes to the flake. +#!/usr/bin/env nix +#!nix shell .#changelog-d-nix --command bash # --- CONFIGURATION --- diff --git a/maintainers/release-process.md b/maintainers/release-process.md index db8b064a514..da6886ea94d 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -27,8 +27,9 @@ release: * Compile the release notes by running ```console + $ export VERSION=X.YY $ git checkout -b release-notes - $ VERSION=X.YY ./maintainers/release-notes + $ ./maintainers/release-notes ``` where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. diff --git a/mk/disable-tests.mk b/mk/disable-tests.mk deleted file mode 100644 index f72f8441256..00000000000 --- a/mk/disable-tests.mk +++ /dev/null @@ -1,12 +0,0 @@ -# This file is only active for `./configure --disable-tests`. -# Running `make check` or `make installcheck` would indicate a mistake in the -# caller. - -installcheck: - @echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make installcheck'." - @exit 1 - -# This currently has little effect. -check: - @echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make check'." - @exit 1 diff --git a/mk/lib.mk b/mk/lib.mk index 3d503364f14..fe0add1c931 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -12,24 +12,7 @@ man-pages := install-tests := install-tests-groups := -ifdef HOST_OS - HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) - ifeq ($(HOST_KERNEL), cygwin) - HOST_CYGWIN = 1 - endif - ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) - HOST_DARWIN = 1 - endif - ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) - HOST_FREEBSD = 1 - endif - ifeq ($(HOST_KERNEL), linux) - HOST_LINUX = 1 - endif - ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) - HOST_SOLARIS = 1 - endif -endif +include mk/platform.mk # Hack to define a literal space. space := @@ -114,6 +97,10 @@ $(foreach test-group, $(install-tests-groups), \ $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval $(test-group).test-group: $(test).test))) +# Include makefiles requiring built programs. +$(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) + + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/libraries.mk b/mk/libraries.mk index 1bc73d7f7a9..b99ba278210 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -3,13 +3,19 @@ libs-list := ifdef HOST_DARWIN SO_EXT = dylib else - ifdef HOST_CYGWIN + ifdef HOST_WINDOWS SO_EXT = dll else SO_EXT = so endif endif +ifdef HOST_UNIX + THREAD_LDFLAGS = -pthread +else + THREAD_LDFLAGS = +endif + # Build a library with symbolic name $(1). The library is defined by # various variables prefixed by ‘$(1)_’: # @@ -59,7 +65,7 @@ define build-library $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - ifdef HOST_CYGWIN + ifdef HOST_WINDOWS $(1)_INSTALL_DIR ?= $$(bindir) else $(1)_INSTALL_DIR ?= $$(libdir) @@ -79,7 +85,7 @@ define build-library endif else ifndef HOST_DARWIN - ifndef HOST_CYGWIN + ifndef HOST_WINDOWS $(1)_LDFLAGS += -Wl,-z,defs endif endif diff --git a/mk/platform.mk b/mk/platform.mk new file mode 100644 index 00000000000..fe960dedf73 --- /dev/null +++ b/mk/platform.mk @@ -0,0 +1,32 @@ +ifdef HOST_OS + HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),) + HOST_MINGW = 1 + HOST_WINDOWS = 1 + endif + ifeq ($(HOST_KERNEL), cygwin) + HOST_CYGWIN = 1 + HOST_WINDOWS = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) + HOST_DARWIN = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) + HOST_FREEBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),) + HOST_NETBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(HOST_KERNEL), linux) + HOST_LINUX = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) + HOST_SOLARIS = 1 + HOST_UNIX = 1 + endif +endif diff --git a/mk/programs.mk b/mk/programs.mk index 6235311e9e3..623caaf556a 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -1,5 +1,11 @@ programs-list := +ifdef HOST_WINDOWS + EXE_EXT = .exe +else + EXE_EXT = +endif + # Build a program with symbolic name $(1). The program is defined by # various variables prefixed by ‘$(1)_’: # @@ -31,7 +37,7 @@ define build-program _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH))) - $(1)_PATH := $$(_d)/$$($(1)_NAME) + $(1)_PATH := $$(_d)/$$($(1)_NAME)$(EXE_EXT) $$(eval $$(call create-dir, $$(_d))) @@ -42,7 +48,7 @@ define build-program ifdef $(1)_INSTALL_DIR - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)$(EXE_EXT) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) diff --git a/mk/templates.mk b/mk/templates.mk index 866bdc17f8b..d5dae61c72b 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -$(buildprefix)%.h: %.h.in +$(buildprefix)%.h: %.h.in $(buildprefix)config.status $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -$(buildprefix)%: %.in +$(buildprefix)%: %.in $(buildprefix)config.status $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/package.nix b/package.nix new file mode 100644 index 00000000000..d1d14d10ea2 --- /dev/null +++ b/package.nix @@ -0,0 +1,390 @@ +{ lib +, stdenv +, releaseTools +, autoconf-archive +, autoreconfHook +, aws-sdk-cpp +, boehmgc +, nlohmann_json +, bison +, boost +, brotli +, bzip2 +, curl +, editline +, readline +, fileset +, flex +, git +, gtest +, jq +, doxygen +, libarchive +, libcpuid +, libgit2 +, libseccomp +, libsodium +, lowdown +, mdbook +, mdbook-linkcheck +, mercurial +, openssh +, openssl +, pkg-config +, rapidcheck +, sqlite +, util-linux +, xz + +, busybox-sandbox-shell ? null + +# Configuration Options +#: +# This probably seems like too many degrees of freedom, but it +# faithfully reflects how the underlying configure + make build system +# work. The top-level flake.nix will choose useful combinations of these +# options to CI. + +, pname ? "nix" + +, versionSuffix ? "" +, officialRelease ? false + +# Whether to build Nix. Useful to skip for tasks like (a) just +# generating API docs or (b) testing existing pre-built versions of Nix +, doBuild ? true + +# Run the unit tests as part of the build. See `installUnitTests` for an +# alternative to this. +, doCheck ? __forDefaults.canRunInstalled + +# Run the functional tests as part of the build. +, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled + +# Check test coverage of Nix. Probably want to use with with at least +# one of `doCHeck` or `doInstallCheck` enabled. +, withCoverageChecks ? false + +# Whether to build the regular manual +, enableManual ? __forDefaults.canRunInstalled + +# Whether to use garbage collection for the Nix language evaluator. +# +# If it is disabled, we just leak memory, but this is not as bad as it +# sounds so long as evaluation just takes places within short-lived +# processes. (When the process exits, the memory is reclaimed; it is +# only leaked *within* the process.) +, enableGC ? true + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" + +# Whether to build the internal API docs, can be done separately from +# everything else. +, enableInternalAPIDocs ? false + +# Whether to install unit tests. This is useful when cross compiling +# since we cannot run them natively during the build, but can do so +# later. +, installUnitTests ? doBuild && !__forDefaults.canExecuteHost + +# For running the functional tests against a pre-built Nix. Probably +# want to use in conjunction with `doBuild = false;`. +, test-daemon ? null +, test-client ? null + +# Avoid setting things that would interfere with a functioning devShell +, forDevShell ? false + +# Not a real argument, just the only way to approximate let-binding some +# stuff for argument defaults. +, __forDefaults ? { + canExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + canRunInstalled = doBuild && __forDefaults.canExecuteHost; + } +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + # selected attributes with defaults, will be used to define some + # things which should instead be gotten via `finalAttrs` in order to + # work with overriding. + attrs = { + inherit doBuild doCheck doInstallCheck; + }; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: let + + inherit (finalAttrs) + doCheck + doInstallCheck + ; + + doBuild = !finalAttrs.dontBuild; + + # Either running the unit tests during the build, or installing them + # to be run later, requiresthe unit tests to be built. + buildUnitTests = doCheck || installUnitTests; + +in { + inherit pname version; + + src = + let + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + in + fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions ([ + # For configure + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + # For make, regardless of what we are building + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ] ++ lib.optionals doBuild [ + ./doc + ./misc + ./precompiled-headers.h + ./src + ./COPYING + ./scripts/local.mk + ] ++ lib.optionals buildUnitTests [ + ./doc/manual + ] ++ lib.optionals enableInternalAPIDocs [ + ./doc/internal-api + # Source might not be compiled, but still must be available + # for Doxygen to gather comments. + ./src + ./tests/unit + ] ++ lib.optionals buildUnitTests [ + ./tests/unit + ] ++ lib.optionals doInstallCheck [ + ./tests/functional + ])); + }; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" ] + ++ lib.optional doBuild "dev" + # If we are doing just build or just docs, the one thing will use + # "out". We only need additional outputs if we are doing both. + ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" + ++ lib.optional installUnitTests "check"; + + nativeBuildInputs = [ + autoconf-archive + autoreconfHook + pkg-config + ] ++ lib.optionals doBuild [ + bison + flex + ] ++ lib.optionals enableManual [ + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + ] ++ lib.optionals (doInstallCheck || enableManual) [ + jq # Also for custom mdBook preprocessor. + ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux + ++ lib.optional enableInternalAPIDocs doxygen + ; + + buildInputs = lib.optionals doBuild [ + boost + brotli + bzip2 + curl + libarchive + libgit2 + libsodium + openssl + sqlite + xz + ({ inherit readline editline; }.${readlineFlavor}) + ] ++ lib.optionals enableMarkdown [ + lowdown + ] ++ lib.optionals buildUnitTests [ + gtest + rapidcheck + ] ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }) + ; + + propagatedBuildInputs = [ + nlohmann_json + ] ++ lib.optional enableGC boehmgc; + + dontBuild = !attrs.doBuild; + doCheck = attrs.doCheck; + + nativeCheckInputs = [ + git + mercurial + openssh + ]; + + disallowedReferences = [ boost ]; + + preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( + '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + '' + lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + '' + ); + + configureFlags = [ + (lib.enableFeature doBuild "build") + (lib.enableFeature buildUnitTests "unit-tests") + (lib.enableFeature doInstallCheck "functional-tests") + (lib.enableFeature enableInternalAPIDocs "internal-api-docs") + (lib.enableFeature enableManual "doc-gen") + (lib.enableFeature enableGC "gc") + (lib.enableFeature enableMarkdown "markdown") + (lib.enableFeature installUnitTests "install-unit-tests") + (lib.withFeatureAs true "readline-flavor" readlineFlavor) + ] ++ lib.optionals (!forDevShell) [ + "--sysconfdir=/etc" + ] ++ lib.optionals installUnitTests [ + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ] ++ lib.optionals (doBuild) [ + "--with-boost=${boost}/lib" + ] ++ lib.optionals (doBuild && stdenv.isLinux) [ + "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" + ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) + "LDFLAGS=-fuse-ld=gold" + ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell" + ; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + + installTargets = lib.optional doBuild "install" + ++ lib.optional enableInternalAPIDocs "internal-api-html"; + + installFlags = "sysconfdir=$(out)/etc"; + + # In this case we are probably just running tests, and so there isn't + # anything to install, we just make an empty directory to signify tests + # succeeded. + installPhase = if finalAttrs.installTargets != [] then null else '' + mkdir -p $out + ''; + + postInstall = lib.optionalString doBuild ( + lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + '' + lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + '' + ) + lib.optionalString enableManual '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products + '' + lib.optionalString enableInternalAPIDocs '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + doInstallCheck = attrs.doInstallCheck; + + installCheckFlags = "sysconfdir=$(out)/etc"; + # Work around buggy detection in stdenv. + installCheckTarget = "installcheck"; + + # Work around weird bug where it doesn't think there is a Makefile. + installCheckPhase = if (!doBuild && doInstallCheck) then '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + '' else null; + + # Needed for tests if we are not doing a build, but testing existing + # built Nix. + preInstallCheck = lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = "nix"; + broken = !(lib.all (a: a) [ + # We cannot run or install unit tests if we don't build them or + # Nix proper (which they depend on). + (installUnitTests -> doBuild) + (doCheck -> doBuild) + # The build process for the manual currently requires extracting + # data from the Nix executable we are trying to document. + (enableManual -> doBuild) + ]); + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; + + dontInstall = false; +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +} // lib.optionalAttrs (test-client != null) { + NIX_CLIENT_PACKAGE = test-client; +}) diff --git a/perl/.yath.rc b/perl/.yath.rc new file mode 100644 index 00000000000..118bf80c821 --- /dev/null +++ b/perl/.yath.rc @@ -0,0 +1,2 @@ +[test] +-I=rel(lib/Nix) diff --git a/perl/default.nix b/perl/default.nix index 4687976a168..7103574c9a9 100644 --- a/perl/default.nix +++ b/perl/default.nix @@ -5,12 +5,12 @@ , nix, curl, bzip2, xz, boost, libsodium, darwin }: -perl.pkgs.toPerlModule (stdenv.mkDerivation { +perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { name = "nix-perl-${nix.version}"; src = fileset.toSource { root = ../.; - fileset = fileset.unions [ + fileset = fileset.unions ([ ../.version ../m4 ../mk @@ -20,7 +20,10 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation { ./configure.ac ./lib ./local.mk - ]; + ] ++ lib.optionals finalAttrs.doCheck [ + ./.yath.rc + ./t + ]); }; nativeBuildInputs = @@ -40,6 +43,13 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation { ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + # `perlPackages.Test2Harness` is marked broken for Darwin + doCheck = !stdenv.isDarwin; + + nativeCheckInputs = [ + perlPackages.Test2Harness + ]; + configureFlags = [ "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" @@ -48,4 +58,4 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation { enableParallelBuilding = true; postUnpack = "sourceRoot=$sourceRoot/perl"; -}) +})) diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index 3e4bbee0ac5..16f2e17c81a 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -12,17 +12,20 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); our @EXPORT = qw( - setVerbosity - isValidPath queryReferences queryPathInfo queryDeriver queryPathHash - queryPathFromHashPart - topoSortPaths computeFSClosure followLinksToStorePath exportPaths importPaths + StoreWrapper + StoreWrapper::new + StoreWrapper::isValidPath StoreWrapper::queryReferences StoreWrapper::queryPathInfo StoreWrapper::queryDeriver StoreWrapper::queryPathHash + StoreWrapper::queryPathFromHashPart + StoreWrapper::topoSortPaths StoreWrapper::computeFSClosure followLinksToStorePath StoreWrapper::exportPaths StoreWrapper::importPaths + StoreWrapper::addToStore StoreWrapper::makeFixedOutputPath + StoreWrapper::derivationFromPath + StoreWrapper::addTempRoot + StoreWrapper::queryRawRealisation + hashPath hashFile hashString convertHash signString checkSignature - addToStore makeFixedOutputPath - derivationFromPath - addTempRoot getBinDir getStoreDir - queryRawRealisation + setVerbosity ); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 82c7db608a0..4a928594bd5 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -12,41 +12,79 @@ #include "realisation.hh" #include "globals.hh" #include "store-api.hh" -#include "crypto.hh" +#include "posix-source-accessor.hh" #include #include - using namespace nix; +static bool libStoreInitialized = false; -static ref store() -{ - static std::shared_ptr _store; - if (!_store) { - try { - initLibStore(); - _store = openStore(); - } catch (Error & e) { - croak("%s", e.what()); - } - } - return ref(_store); -} - +struct StoreWrapper { + ref store; +}; MODULE = Nix::Store PACKAGE = Nix::Store PROTOTYPES: ENABLE +TYPEMAP: < _store; + try { + if (!libStoreInitialized) { + initLibStore(); + libStoreInitialized = true; + } + if (items == 1) { + _store = openStore(); + RETVAL = new StoreWrapper { + .store = ref{_store} + }; + } else { + RETVAL = new StoreWrapper { + .store = openStore(s) + }; + } + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + void init() CODE: - store(); + if (!libStoreInitialized) { + initLibStore(); + libStoreInitialized = true; + } void setVerbosity(int level) @@ -54,10 +92,11 @@ void setVerbosity(int level) verbosity = (Verbosity) level; -int isValidPath(char * path) +int +StoreWrapper::isValidPath(char * path) CODE: try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + RETVAL = THIS->store->isValidPath(THIS->store->parseStorePath(path)); } catch (Error & e) { croak("%s", e.what()); } @@ -65,52 +104,56 @@ int isValidPath(char * path) RETVAL -SV * queryReferences(char * path) +SV * +StoreWrapper::queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + for (auto & i : THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->references) + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryPathHash(char * path) +SV * +StoreWrapper::queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); + auto s = THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryDeriver(char * path) +SV * +StoreWrapper::queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); if (!info->deriver) XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryPathInfo(char * path, int base32) +SV * +StoreWrapper::queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); if (!info->deriver) XPUSHs(&PL_sv_undef); else - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); for (auto & i : info->references) - av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); + av_push(refs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); for (auto & i : info->sigs) @@ -120,10 +163,11 @@ SV * queryPathInfo(char * path, int base32) croak("%s", e.what()); } -SV * queryRawRealisation(char * outputId) +SV * +StoreWrapper::queryRawRealisation(char * outputId) PPCODE: try { - auto realisation = store()->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); if (realisation) XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0))); else @@ -133,46 +177,50 @@ SV * queryRawRealisation(char * outputId) } -SV * queryPathFromHashPart(char * hashPart) +SV * +StoreWrapper::queryPathFromHashPart(char * hashPart) PPCODE: try { - auto path = store()->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0))); + auto path = THIS->store->queryPathFromHashPart(hashPart); + XPUSHs(sv_2mortal(newSVpv(path ? THIS->store->printStorePath(*path).c_str() : "", 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * computeFSClosure(int flipDirection, int includeOutputs, ...) +SV * +StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { StorePathSet paths; for (int n = 2; n < items; ++n) - store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); + THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); for (auto & i : paths) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * topoSortPaths(...) +SV * +StoreWrapper::topoSortPaths(...) PPCODE: try { StorePathSet paths; - for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); - auto sorted = store()->topoSortPaths(paths); + for (int n = 0; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); + auto sorted = THIS->store->topoSortPaths(paths); for (auto & i : sorted) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * followLinksToStorePath(char * path) +SV * +StoreWrapper::followLinksToStorePath(char * path) CODE: try { - RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0); + RETVAL = newSVpv(THIS->store->printStorePath(THIS->store->followLinksToStorePath(path)).c_str(), 0); } catch (Error & e) { croak("%s", e.what()); } @@ -180,32 +228,38 @@ SV * followLinksToStorePath(char * path) RETVAL -void exportPaths(int fd, ...) +void +StoreWrapper::exportPaths(int fd, ...) PPCODE: try { StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); + for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); - store()->exportPaths(paths, sink); + THIS->store->exportPaths(paths, sink); } catch (Error & e) { croak("%s", e.what()); } -void importPaths(int fd, int dontCheckSigs) +void +StoreWrapper::importPaths(int fd, int dontCheckSigs) PPCODE: try { FdSource source(fd); - store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); + THIS->store->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); } catch (Error & e) { croak("%s", e.what()); } -SV * hashPath(char * algo, int base32, char * path) +SV * +hashPath(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashPath(parseHashAlgo(algo), path).first; + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path); + Hash h = hashPath( + accessor, canonPath, + FileIngestionMethod::Recursive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -277,60 +331,67 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) RETVAL -SV * addToStore(char * srcPath, int recursive, char * algo) +SV * +StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo)); - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(srcPath); + auto path = THIS->store->addToStore( + std::string(baseNameOf(srcPath)), + accessor, canonPath, + method, parseHashAlgo(algo)); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) +SV * +StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { auto h = Hash::parseAny(hash, parseHashAlgo(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { .method = method, .hash = h, .references = {}, }); - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * derivationFromPath(char * drvPath) +SV * +StoreWrapper::derivationFromPath(char * drvPath) PREINIT: HV *hash; CODE: try { - Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath)); + Derivation drv = THIS->store->derivationFromPath(THIS->store->parseStorePath(drvPath)); hash = newHV(); HV * outputs = newHV(); - for (auto & i : drv.outputsAndOptPaths(*store())) { + for (auto & i : drv.outputsAndOptPaths(*THIS->store)) { hv_store( outputs, i.first.c_str(), i.first.size(), !i.second.second ? newSV(0) /* null value */ - : newSVpv(store()->printStorePath(*i.second.second).c_str(), 0), + : newSVpv(THIS->store->printStorePath(*i.second.second).c_str(), 0), 0); } hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); for (auto & i : drv.inputDrvs.map) - av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second + av_push(inputDrvs, newSVpv(THIS->store->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); AV * inputSrcs = newAV(); for (auto & i : drv.inputSrcs) - av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0)); + av_push(inputSrcs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); @@ -354,10 +415,11 @@ SV * derivationFromPath(char * drvPath) RETVAL -void addTempRoot(char * storePath) +void +StoreWrapper::addTempRoot(char * storePath) PPCODE: try { - store()->addTempRoot(store()->parseStorePath(storePath)); + THIS->store->addTempRoot(THIS->store->parseStorePath(storePath)); } catch (Error & e) { croak("%s", e.what()); } diff --git a/perl/local.mk b/perl/local.mk index 0eae651d8b6..ed4764eb96b 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -41,3 +41,6 @@ Store_FORCE_INSTALL = 1 Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config + +check: all + yath test diff --git a/perl/t/init.t b/perl/t/init.t new file mode 100644 index 00000000000..80197e01376 --- /dev/null +++ b/perl/t/init.t @@ -0,0 +1,13 @@ +use strict; +use warnings; +use Test2::V0; + +use Nix::Store; + +my $s = new Nix::Store("dummy://"); + +my $res = $s->isValidPath("/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + +ok(!$res, "should not have path"); + +done_testing; diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix new file mode 100644 index 00000000000..104189b0c81 --- /dev/null +++ b/scripts/binary-tarball.nix @@ -0,0 +1,84 @@ +{ runCommand +, system +, buildPackages +, cacert +, nix +}: + +let + + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ nix cacert ]; + }; + + inherit (nix) version; + + env = { + #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + }; + +in + +runCommand "nix-binary-tarball-${version}" env '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) +'' diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 0326d34150e..24c9052f91c 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,11 +3,13 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}" +export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" + readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist # create by default; set 0 to DIY, use a symlink, etc. readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default -NIX_FIRST_BUILD_UID="301" -NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" # caution: may update times on / if not run as normal non-root user read_only_root() { @@ -100,7 +102,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m "u=rw,go=r" "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a08f62333dc..ad3ee888138 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,9 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}" readonly NIX_BUILD_GROUP_NAME="nixbld" -# darwin installer needs to override these -NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" -NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" +# each system specific installer must set these: +# NIX_FIRST_BUILD_UID +# NIX_BUILD_USER_NAME_TEMPLATE # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -707,6 +707,12 @@ EOF fi } +check_required_system_specific_settings() { + if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then + failure "Internal error: System specific installer for $(uname) ($1) does not export required settings." + fi +} + welcome_to_nix() { local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}") @@ -726,7 +732,9 @@ manager. This will happen in a few stages: if you are ready to continue. 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT} - that the Nix daemon uses to run builds. + that the Nix daemon uses to run builds. To create system users + in a different range, exit and run this tool again with + NIX_FIRST_BUILD_UID set. 4. Perform the basic installation of the Nix files daemon. @@ -968,13 +976,16 @@ main() { if is_os_darwin; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + check_required_system_specific_settings "install-darwin-multi-user.sh" elif is_os_linux; then # shellcheck source=./install-systemd-multi-user.sh . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also + check_required_system_specific_settings "install-systemd-multi-user.sh" else failure "Sorry, I don't know what to do on $(uname)" fi + welcome_to_nix if ! is_root; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 07b34033a51..202a9bb548b 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -3,6 +3,10 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" +export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" + readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service diff --git a/scripts/installer.nix b/scripts/installer.nix new file mode 100644 index 00000000000..cc7759c2c8e --- /dev/null +++ b/scripts/installer.nix @@ -0,0 +1,36 @@ +{ lib +, runCommand +, nix +, tarballs +}: + +runCommand "installer-script" { + buildInputs = [ nix ]; +} '' + mkdir -p $out/nix-support + + # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + + substitute ${./install.in} $out/install \ + ${lib.concatMapStrings + (tarball: let + inherit (tarball.stdenv.hostPlatform) system; + in '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ + '' + ) + tarballs + } --replace '@nixVersion@' ${nix.version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products +'' diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index d69d3a0c263..118468477dd 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -137,11 +137,8 @@ static int main_build_remote(int argc, char * * argv) for (auto & m : machines) { debug("considering building on remote machine '%s'", m.storeUri); - if (m.enabled - && (neededSystem == "builtin" - || std::find(m.systemTypes.begin(), - m.systemTypes.end(), - neededSystem) != m.systemTypes.end()) && + if (m.enabled && + m.systemSupported(neededSystem) && m.allSupported(requiredFeatures) && m.mandatoryMet(requiredFeatures)) { @@ -205,7 +202,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = hintformat(errorText); + auto error = HintFmt(errorText); error % drvstr % neededSystem @@ -214,7 +211,7 @@ static int main_build_remote(int argc, char * * argv) for (auto & m : machines) error - % concatStringsSep>(", ", m.systemTypes) + % concatStringsSep(", ", m.systemTypes) % m.maxJobs % concatStringsSep(", ", m.supportedFeatures) % concatStringsSep(", ", m.mandatoryFeatures); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 8e2efc7c31b..c5eb93c5d7e 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -12,9 +12,9 @@ namespace nix { bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ { \ const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields1 = std::tie(*me->drvPath, me->FIELD); \ me = &other; \ - auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields2 = std::tie(*me->drvPath, me->FIELD); \ return fields1 COMPARATOR fields2; \ } #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 2beb2af26af..c28d598b75a 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -157,7 +157,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) for (auto & i : autoArgs) { auto v = state.allocValue(); if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(CanonPath::fromCwd()))); + state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); else v->mkString(((std::string_view) i.second).substr(1)); res.insert(state.symbols.create(i.first), v); @@ -165,7 +165,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) +SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto accessor = fetchers::downloadTarball( @@ -187,7 +187,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDi } else - return state.rootPath(CanonPath(s, baseDir)); + return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 4b403d93674..2eb63e15d50 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -29,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); +SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 619d3673f39..6bf36bd647b 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,6 @@ #include "editor-for.hh" #include "environment-variables.hh" +#include "source-path.hh" namespace nix { @@ -16,7 +17,7 @@ Strings editorFor(const SourcePath & file, uint32_t line) editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) args.push_back(fmt("+%d", line)); - args.push_back(path->abs()); + args.push_back(path->string()); return args; } diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index fbf4307c90e..8acd7011e69 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -2,7 +2,7 @@ ///@file #include "types.hh" -#include "input-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 06e507872b5..3ec1c161477 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -58,22 +58,22 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() Bindings & autoArgs = *cmd.getAutoArgs(*state); - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); + PackageInfos packageInfos; + getDerivations(*state, *v, "", autoArgs, packageInfos, false); // Backward compatibility hack: group results by drvPath. This // helps keep .all output together. std::map byDrvPath; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); + for (auto & packageInfo : packageInfos) { + auto drvPath = packageInfo.queryDrvPath(); if (!drvPath) throw Error("'%s' is not a derivation", what()); auto newOutputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set outputsToInstall; - for (auto & output : drvInfo.queryOutputs(false, true)) + for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); return OutputsSpec::Names { std::move(outputsToInstall) }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 2f428cb7e4b..ddec7537bab 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -52,7 +52,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); assert(aOutputs); - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); + state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); return aOutputs->value; } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 08ad35105bc..c8a3e1b2175 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -1,5 +1,6 @@ #include "installable-value.hh" #include "eval-cache.hh" +#include "fetch-to-store.hh" namespace nix { @@ -44,7 +45,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = v.path().fetchToStore(state->store); + auto storePath = fetchToStore(*state->store, v.path()); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index 3138ce8ecee..f300d392bb5 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -6,7 +6,7 @@ namespace nix { -struct DrvInfo; +struct PackageInfo; struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6baa82544ad..6db9bf9a129 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -147,7 +147,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = getUnfilteredRootPath(CanonPath::fromCwd(lockFilePath)); + lockFlags.referenceLockFilePath = getUnfilteredRootPath(CanonPath(absPath(lockFilePath))); }}, .completer = completePath }); @@ -158,7 +158,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.outputLockFilePath = CanonPath::fromCwd(lockFilePath); + lockFlags.outputLockFilePath = lockFilePath; }}, .completer = completePath }); @@ -488,10 +488,11 @@ Installables SourceExprCommand::parseInstallables( state->eval(e, *vFile); } else if (file) { - state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + auto dir = absPath(getCommandBaseDir()); + state->evalFile(lookupFileArg(*state, *file, &dir), *vFile); } else { - CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + Path dir = absPath(getCommandBaseDir()); auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } @@ -716,7 +717,7 @@ BuiltPaths Installable::toBuiltPaths( } } -StorePathSet Installable::toStorePaths( +StorePathSet Installable::toStorePathSet( ref evalStore, ref store, Realise mode, OperateOn operateOn, @@ -730,13 +731,27 @@ StorePathSet Installable::toStorePaths( return outPaths; } +StorePaths Installable::toStorePaths( + ref evalStore, + ref store, + Realise mode, OperateOn operateOn, + const Installables & installables) +{ + StorePaths outPaths; + for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { + auto thisOutPaths = path.outPaths(); + outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end()); + } + return outPaths; +} + StorePath Installable::toStorePath( ref evalStore, ref store, Realise mode, OperateOn operateOn, ref installable) { - auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); + auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable}); if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e087f935c85..bf5759230f3 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -12,7 +12,7 @@ namespace nix { -struct DrvInfo; +struct PackageInfo; enum class Realise { /** @@ -165,7 +165,14 @@ struct Installable const Installables & installables, BuildMode bMode = bmNormal); - static std::set toStorePaths( + static std::set toStorePathSet( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const Installables & installables); + + static std::vector toStorePaths( ref evalStore, ref store, Realise mode, diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index afd35af0842..abb7459a7d0 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc) libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread +libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) libcmd_LIBS = libstore libutil libexpr libmain libfetchers diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 8b3bbc1b5e8..a4e3c5a77af 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -4,12 +4,15 @@ #include "terminal.hh" #include +#if HAVE_LOWDOWN #include +#endif namespace nix { std::string renderMarkdownToTerminal(std::string_view markdown) { +#if HAVE_LOWDOWN int windowWidth = getWindowSize().second; struct lowdown_opts opts { @@ -48,6 +51,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); +#else + return std::string(markdown); +#endif } } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 0986296ad45..03602e170b5 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -5,7 +5,7 @@ #include -#ifdef READLINE +#ifdef USE_READLINE #include #include #else @@ -93,9 +93,18 @@ struct NixRepl void evalString(std::string s, Value & v); void loadDebugTraceEnv(DebugTrace & dt); - typedef std::set ValuesSeen; - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); + void printValue(std::ostream & str, + Value & v, + unsigned int maxDepth = std::numeric_limits::max()) + { + ::nix::printValue(*state, str, v, PrintOptions { + .ansiColors = true, + .force = true, + .derivationPaths = true, + .maxDepth = maxDepth, + .prettyIndent = 2 + }); + } }; std::string removeWhitespace(std::string s) @@ -112,7 +121,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, refstaticBaseEnv.get())) + , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) , historyFile(getDataDir() + "/nix/repl-history") { } @@ -221,10 +230,10 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi // prefer direct pos, but if noPos then try the expr. auto pos = dt.pos ? dt.pos - : static_cast>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]); + : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]; if (pos) { - out << pos; + out << *pos; if (auto loc = pos->getCodeLines()) { out << "\n"; printCodeLines(out, "", *pos, *loc); @@ -235,10 +244,19 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi return out; } +static bool isFirstRepl = true; + void NixRepl::mainLoop() { - std::string error = ANSI_RED "error:" ANSI_NORMAL " "; - notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); + if (isFirstRepl) { + std::string_view debuggerNotice = ""; + if (state->debugRepl) { + debuggerNotice = " debugger"; + } + notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice); + } + + isFirstRepl = false; loadFiles(); @@ -246,17 +264,17 @@ void NixRepl::mainLoop() rl_readline_name = "nix-repl"; try { createDirs(dirOf(historyFile)); - } catch (SysError & e) { + } catch (SystemError & e) { logWarning(e.info()); } -#ifndef READLINE +#ifndef USE_READLINE el_hist_size = 1000; #endif read_history(historyFile.c_str()); auto oldRepl = curRepl; curRepl = this; Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef READLINE +#ifndef USE_READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); #endif @@ -414,8 +432,6 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore parse errors. } catch (EvalError & e) { // Quietly ignore evaluation errors. - } catch (UndefinedVarError & e) { - // Quietly ignore undefined variable errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. } @@ -442,10 +458,10 @@ static bool isVarName(std::string_view s) StorePath NixRepl::getDerivationPath(Value & v) { - auto drvInfo = getDerivation(*state, v, false); - if (!drvInfo) + auto packageInfo = getDerivation(*state, v, false); + if (!packageInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); - auto drvPath = drvInfo->queryDrvPath(); + auto drvPath = packageInfo->queryDrvPath(); if (!drvPath) throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); if (!state->store->isValidPath(*drvPath)) @@ -708,7 +724,8 @@ bool NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v, 1000000000) << std::endl; + printValue(std::cout, v); + std::cout << std::endl; } else if (command == ":q" || command == ":quit") { @@ -770,7 +787,8 @@ bool NixRepl::processLine(std::string line) } else { Value v; evalString(line, v); - printValue(std::cout, v, 1) << std::endl; + printValue(std::cout, v, 1); + std::cout << std::endl; } } @@ -880,7 +898,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v) Expr * NixRepl::parseString(std::string s) { - return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv); + return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } @@ -888,145 +906,7 @@ void NixRepl::evalString(std::string s, Value & v) { Expr * e = parseString(s); e->eval(*state, *env, v); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); -} - - -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) -{ - ValuesSeen seen; - return printValue(str, v, maxDepth, seen); -} - - - - -// FIXME: lot of cut&paste from Nix's eval.cc. -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) -{ - str.flush(); - checkInterrupt(); - - state->forceValue(v, [&]() { return v.determinePos(noPos); }); - - switch (v.type()) { - - case nInt: - str << ANSI_CYAN << v.integer << ANSI_NORMAL; - break; - - case nBool: - str << ANSI_CYAN; - printLiteralBool(str, v.boolean); - str << ANSI_NORMAL; - break; - - case nString: - str << ANSI_WARNING; - printLiteralString(str, v.string_view()); - str << ANSI_NORMAL; - break; - - case nPath: - str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping? - break; - - case nNull: - str << ANSI_CYAN "null" ANSI_NORMAL; - break; - - case nAttrs: { - seen.insert(&v); - - bool isDrv = state->isDerivation(v); - - if (isDrv) { - str << "«derivation "; - Bindings::iterator i = v.attrs->find(state->sDrvPath); - NixStringContext context; - if (i != v.attrs->end()) - str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); - else - str << "???"; - str << "»"; - } - - else if (maxDepth > 0) { - str << "{ "; - - typedef std::map Sorted; - Sorted sorted; - for (auto & i : *v.attrs) - sorted.emplace(state->symbols[i.name], i.value); - - for (auto & i : sorted) { - printAttributeName(str, i.first); - str << " = "; - if (seen.count(i.second)) - str << "«repeated»"; - else - try { - printValue(str, *i.second, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; - } - str << "; "; - } - - str << "}"; - } else - str << "{ ... }"; - - break; - } - - case nList: - seen.insert(&v); - - str << "[ "; - if (maxDepth > 0) - for (auto elem : v.listItems()) { - if (seen.count(elem)) - str << "«repeated»"; - else - try { - printValue(str, *elem, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; - } - str << " "; - } - else - str << "... "; - str << "]"; - break; - - case nFunction: - if (v.isLambda()) { - std::ostringstream s; - s << state->positions[v.lambda.fun->pos]; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - } else if (v.isPrimOp()) { - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - } else if (v.isPrimOpApp()) { - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; - } else { - abort(); - } - break; - - case nFloat: - str << v.fpoint; - break; - - case nThunk: - case nExternal: - default: - str << ANSI_RED "«unknown»" ANSI_NORMAL; - break; - } - - return str; + state->forceValue(v, v.determinePos(noPos)); } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 7481a2232da..d6befd362e4 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -65,10 +65,10 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!attrIndex) { if (v->type() != nAttrs) - throw TypeError( + state.error( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, - showType(*v)); + showType(*v)).debugThrow(); if (attr.empty()) throw Error("empty attribute name in selection path '%1%'", attrPath); @@ -88,10 +88,10 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin else { if (!v->isList()) - throw TypeError( + state.error( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, - showType(*v)); + showType(*v)).debugThrow(); if (*attrIndex >= v->listSize()) throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 5808d58b6b4..2fc69e79674 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -491,7 +491,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (forceErrors) debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); else - throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name)); } else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); @@ -500,7 +500,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro // evaluate to see whether 'name' exists } else return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } @@ -508,7 +508,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (v.type() != nAttrs) return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); auto attr = v.attrs->get(name); @@ -574,14 +574,14 @@ std::string AttrCursor::getString() debug("using cached string attribute '%s'", getAttrPathStr()); return s->first; } else - root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nString && v.type() != nPath) - root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); return v.type() == nString ? v.c_str() : v.path().to_string(); } @@ -616,7 +616,7 @@ string_t AttrCursor::getStringWithContext() return *s; } } else - root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } @@ -630,7 +630,7 @@ string_t AttrCursor::getStringWithContext() else if (v.type() == nPath) return {v.path().to_string(), {}}; else - root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); } bool AttrCursor::getBool() @@ -643,14 +643,14 @@ bool AttrCursor::getBool() debug("using cached Boolean attribute '%s'", getAttrPathStr()); return *b; } else - root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nBool) - root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); return v.boolean; } @@ -665,14 +665,14 @@ NixInt AttrCursor::getInt() debug("using cached integer attribute '%s'", getAttrPathStr()); return i->x; } else - throw TypeError("'%s' is not an integer", getAttrPathStr()); + root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nInt) - throw TypeError("'%s' is not an integer", getAttrPathStr()); + root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); return v.integer; } @@ -687,7 +687,7 @@ std::vector AttrCursor::getListOfStrings() debug("using cached list of strings attribute '%s'", getAttrPathStr()); return *l; } else - throw TypeError("'%s' is not a list of strings", getAttrPathStr()); + root->state.error("'%s' is not a list of strings", getAttrPathStr()).debugThrow(); } } @@ -697,7 +697,7 @@ std::vector AttrCursor::getListOfStrings() root->state.forceValue(v, noPos); if (v.type() != nList) - throw TypeError("'%s' is not a list", getAttrPathStr()); + root->state.error("'%s' is not a list", getAttrPathStr()).debugThrow(); std::vector res; @@ -720,14 +720,14 @@ std::vector AttrCursor::getAttrs() debug("using cached attrset attribute '%s'", getAttrPathStr()); return *attrs; } else - root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nAttrs) - root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); std::vector attrs; for (auto & attr : *getValue().attrs) diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc new file mode 100644 index 00000000000..f4cdeec5c9f --- /dev/null +++ b/src/libexpr/eval-error.cc @@ -0,0 +1,113 @@ +#include "eval-error.hh" +#include "eval.hh" +#include "value.hh" + +namespace nix { + +template +EvalErrorBuilder & EvalErrorBuilder::withExitStatus(unsigned int exitStatus) +{ + error.withExitStatus(exitStatus); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::atPos(PosIdx pos) +{ + error.err.pos = error.state.positions[pos]; + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::atPos(Value & value, PosIdx fallback) +{ + return atPos(value.determinePos(fallback)); +} + +template +EvalErrorBuilder & EvalErrorBuilder::withTrace(PosIdx pos, const std::string_view text) +{ + error.err.traces.push_front( + Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = false}); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withFrameTrace(PosIdx pos, const std::string_view text) +{ + error.err.traces.push_front( + Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = true}); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withSuggestions(Suggestions & s) +{ + error.err.suggestions = s; + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withFrame(const Env & env, const Expr & expr) +{ + // NOTE: This is abusing side-effects. + // TODO: check compatibility with nested debugger calls. + // TODO: What side-effects?? + error.state.debugTraces.push_front(DebugTrace{ + .pos = error.state.positions[expr.getPos()], + .expr = expr, + .env = env, + .hint = HintFmt("Fake frame for debugging purposes"), + .isError = true}); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::addTrace(PosIdx pos, HintFmt hint, bool frame) +{ + error.addTrace(error.state.positions[pos], hint, frame); + return *this; +} + +template +template +EvalErrorBuilder & +EvalErrorBuilder::addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs) +{ + + addTrace(error.state.positions[pos], HintFmt(std::string(formatString), formatArgs...)); + return *this; +} + +template +void EvalErrorBuilder::debugThrow() +{ + if (error.state.debugRepl && !error.state.debugTraces.empty()) { + const DebugTrace & last = error.state.debugTraces.front(); + const Env * env = &last.env; + const Expr * expr = &last.expr; + error.state.runDebugRepl(&error, *env, *expr); + } + + // `EvalState` is the only class that can construct an `EvalErrorBuilder`, + // and it does so in dynamic storage. This is the final method called on + // any such instance and must delete itself before throwing the underlying + // error. + auto error = std::move(this->error); + delete this; + + throw error; +} + +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; + +} diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh new file mode 100644 index 00000000000..392902ad22a --- /dev/null +++ b/src/libexpr/eval-error.hh @@ -0,0 +1,104 @@ +#pragma once + +#include + +#include "error.hh" +#include "pos-idx.hh" + +namespace nix { + +struct Env; +struct Expr; +struct Value; + +class EvalState; +template +class EvalErrorBuilder; + +class EvalError : public Error +{ + template + friend class EvalErrorBuilder; +public: + EvalState & state; + + EvalError(EvalState & state, ErrorInfo && errorInfo) + : Error(errorInfo) + , state(state) + { + } + + template + explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs) + : Error(formatString, formatArgs...) + , state(state) + { + } +}; + +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, EvalError); +MakeError(MissingArgumentError, EvalError); +MakeError(CachedEvalError, EvalError); +MakeError(InfiniteRecursionError, EvalError); + +struct InvalidPathError : public EvalError +{ +public: + Path path; + InvalidPathError(EvalState & state, const Path & path) + : EvalError(state, "path '%s' is not valid", path) + { + } +}; + +/** + * `EvalErrorBuilder`s may only be constructed by `EvalState`. The `debugThrow` + * method must be the final method in any such `EvalErrorBuilder` usage, and it + * handles deleting the object. + */ +template +class EvalErrorBuilder final +{ + friend class EvalState; + + template + explicit EvalErrorBuilder(EvalState & state, const Args &... args) + : error(T(state, args...)) + { + } + +public: + T error; + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withExitStatus(unsigned int exitStatus); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & atPos(PosIdx pos); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & atPos(Value & value, PosIdx fallback = noPos); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withSuggestions(Suggestions & s); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withFrame(const Env & e, const Expr & ex); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & addTrace(PosIdx pos, HintFmt hint, bool frame = false); + + template + [[nodiscard, gnu::noinline]] EvalErrorBuilder & + addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs); + + /** + * Delete the `EvalErrorBuilder` and throw the underlying exception. + */ + [[gnu::noinline, gnu::noreturn]] void debugThrow(); +}; + +} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c37b1d62bb1..03320c7c977 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,7 +1,9 @@ #pragma once ///@file +#include "print.hh" #include "eval.hh" +#include "eval-error.hh" namespace nix { @@ -73,8 +75,6 @@ Env & EvalState::allocEnv(size_t size) #endif env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->type = Env::Plain; - /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; @@ -83,13 +83,6 @@ Env & EvalState::allocEnv(size_t size) [[gnu::always_inline]] void EvalState::forceValue(Value & v, const PosIdx pos) -{ - forceValue(v, [&]() { return pos; }); -} - - -template -void EvalState::forceValue(Value & v, Callable getPos) { if (v.isThunk()) { Env * env = v.thunk.env; @@ -100,15 +93,12 @@ void EvalState::forceValue(Value & v, Callable getPos) expr->eval(*this, *env, v); } catch (...) { v.mkThunk(env, expr); + tryFixupBlackHolePos(v, pos); throw; } } - else if (v.isApp()) { - PosIdx pos = getPos(); + else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, pos); - } - else if (v.isBlackhole()) - error("infinite recursion encountered").atPos(getPos()).template debugThrow(); } @@ -126,7 +116,11 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e PosIdx pos = getPos(); forceValue(v, pos); if (v.type() != nAttrs) { - error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); + error( + "expected a set but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withTrace(pos, errorCtx).debugThrow(); } } @@ -136,7 +130,11 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e { forceValue(v, pos); if (!v.isList()) { - error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); + error( + "expected a list but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withTrace(pos, errorCtx).debugThrow(); } } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 444a7d7d6dc..2ccbe327fad 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -89,6 +89,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url) return std::string(url); } +const std::string & EvalSettings::getCurrentSystem() +{ + const auto & evalSystem = currentSystem.get(); + return evalSystem != "" ? evalSystem : settings.thisSystem.get(); +} + EvalSettings evalSettings; static GlobalConfig::Register rEvalSettings(&evalSettings); diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index db2971acb77..2f6c12d4545 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -27,6 +27,26 @@ struct EvalSettings : Config [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; + Setting currentSystem{ + this, "", "eval-system", + R"( + This option defines + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + in the Nix language if it is set as a non-empty string. + Otherwise, if it is defined as the empty string (the default), the value of the + [`system` ](#conf-system) + configuration setting is used instead. + + Unlike `system`, this setting does not change what kind of derivations can be built locally. + This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system. + )"}; + + /** + * Implements the `eval-system` vs `system` defaulting logic + * described for `eval-system`. + */ + const std::string & getCurrentSystem(); + Setting restrictEval{ this, false, "restrict-eval", R"( @@ -68,6 +88,11 @@ struct EvalSettings : Config evaluation mode. For example, when set to `https://github.com/NixOS`, builtin functions such as `fetchGit` are allowed to access `https://github.com/NixOS/patchelf.git`. + + Access is granted when + - the URI is equal to the prefix, + - or the URI is a subpath of the prefix, + - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme. )"}; Setting traceFunctionCalls{this, false, "trace-function-calls", @@ -99,6 +124,9 @@ struct EvalSettings : Config Setting traceVerbose{this, false, "trace-verbose", "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; + + Setting maxCallDepth{this, 10000, "max-call-depth", + "The maximum function call depth to allow before erroring."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 01ca1451f16..ed4d07b2a3a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2,6 +2,7 @@ #include "eval-settings.hh" #include "hash.hh" #include "primops.hh" +#include "print-options.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -18,6 +19,11 @@ #include "memory-input-accessor.hh" #include "signals.hh" #include "gc-small-vector.hh" +#include "url.hh" +#include "fetch-to-store.hh" +#include "tarball.hh" +#include "flake/flakeref.hh" +#include "parser-tab.hh" #include #include @@ -27,9 +33,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -103,116 +109,23 @@ RootValue allocRootValue(Value * v) #endif } -void Value::print(const SymbolTable &symbols, std::ostream &str, - std::set *seen, int depth) const - -{ - checkInterrupt(); - - if (depth <= 0) { - str << "«too deep»"; - return; - } - switch (internalType) { - case tInt: - str << integer; - break; - case tBool: - printLiteralBool(str, boolean); - break; - case tString: - printLiteralString(str, string_view()); - break; - case tPath: - str << path().to_string(); // !!! escaping? - break; - case tNull: - str << "null"; - break; - case tAttrs: { - if (seen && !attrs->empty() && !seen->insert(attrs).second) - str << "«repeated»"; - else { - str << "{ "; - for (auto & i : attrs->lexicographicOrder(symbols)) { - str << symbols[i->name] << " = "; - i->value->print(symbols, str, seen, depth - 1); - str << "; "; - } - str << "}"; - } - break; - } - case tList1: - case tList2: - case tListN: - if (seen && listSize() && !seen->insert(listElems()).second) - str << "«repeated»"; - else { - str << "[ "; - for (auto v2 : listItems()) { - if (v2) - v2->print(symbols, str, seen, depth - 1); - else - str << "(nullptr)"; - str << " "; - } - str << "]"; - } - break; - case tThunk: - case tApp: - str << ""; - break; - case tLambda: - str << ""; - break; - case tPrimOp: - str << ""; - break; - case tPrimOpApp: - str << ""; - break; - case tExternal: - str << *external; - break; - case tFloat: - str << fpoint; - break; - case tBlackhole: - // Although we know for sure that it's going to be an infinite recursion - // when this value is accessed _in the current context_, it's likely - // that the user will misinterpret a simpler «infinite recursion» output - // as a definitive statement about the value, while in fact it may be - // a valid value after `builtins.trace` and perhaps some other steps - // have completed. - str << "«potential infinite recursion»"; - break; - default: - printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); - abort(); - } -} - -void Value::print(const SymbolTable &symbols, std::ostream &str, - bool showRepeated, int depth) const { - std::set seen; - print(symbols, str, showRepeated ? nullptr : &seen, depth); -} - // Pretty print types for assertion errors std::ostream & operator << (std::ostream & os, const ValueType t) { os << showType(t); return os; } -std::string printValue(const EvalState & state, const Value & v) +std::string printValue(EvalState & state, Value & v) { std::ostringstream out; - v.print(state.symbols, out); + v.print(state, out); return out.str(); } +void Value::print(EvalState & state, std::ostream & str, PrintOptions options) +{ + printValue(state, str, *this, options); +} const Value * getPrimOp(const Value &v) { const Value * primOp = &v; @@ -255,9 +168,8 @@ std::string showType(const Value & v) case tPrimOpApp: return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); - case tThunk: return "a thunk"; + case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk"; case tApp: return "a function application"; - case tBlackhole: return "a black hole"; default: return std::string(showType(v.type())); } @@ -427,46 +339,6 @@ void initGC() gcInitialised = true; } - -ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) -{ - info.errPos = state.positions[pos]; - return *this; -} - -ErrorBuilder & ErrorBuilder::withTrace(PosIdx pos, const std::string_view text) -{ - info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = false }); - return *this; -} - -ErrorBuilder & ErrorBuilder::withFrameTrace(PosIdx pos, const std::string_view text) -{ - info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = true }); - return *this; -} - -ErrorBuilder & ErrorBuilder::withSuggestions(Suggestions & s) -{ - info.suggestions = s; - return *this; -} - -ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) -{ - // NOTE: This is abusing side-effects. - // TODO: check compatibility with nested debugger calls. - state.debugTraces.push_front(DebugTrace { - .pos = nullptr, - .expr = expr, - .env = env, - .hint = hintformat("Fake frame for debugging purposes"), - .isError = true - }); - return *this; -} - - EvalState::EvalState( const SearchPath & _searchPath, ref store, @@ -508,18 +380,28 @@ EvalState::EvalState( , sPath(symbols.create("path")) , sPrefix(symbols.create("prefix")) , sOutputSpecified(symbols.create("outputSpecified")) + , exprSymbols{ + .sub = symbols.create("__sub"), + .lessThan = symbols.create("__lessThan"), + .mul = symbols.create("__mul"), + .div = symbols.create("__div"), + .or_ = symbols.create("or"), + .findFile = symbols.create("__findFile"), + .nixPath = symbols.create("__nixPath"), + .body = symbols.create("body"), + } , repair(NoRepair) , emptyBindings(0) , rootFS( evalSettings.restrictEval || evalSettings.pureEval - ? ref(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {}, + ? ref(AllowListInputAccessor::create(makeFSInputAccessor(), {}, [](const CanonPath & path) -> RestrictedPathError { auto modeInformation = evalSettings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) - : makeFSInputAccessor(CanonPath::root)) + : makeFSInputAccessor()) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -543,7 +425,7 @@ EvalState::EvalState( #endif , virtualPathMarker(settings.nixStore + "/virtual00000000000000000") , baseEnv(allocEnv(128)) - , staticBaseEnv{std::make_shared(false, nullptr)} + , staticBaseEnv{std::make_shared(nullptr, nullptr)} { corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); @@ -554,6 +436,8 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); + vEmptyList.mkList(0); + /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { for (auto & i : _searchPath.elements) @@ -600,21 +484,45 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } -void EvalState::checkURI(const std::string & uri) +inline static bool isJustSchemePrefix(std::string_view prefix) { - if (!evalSettings.restrictEval) return; + return + !prefix.empty() + && prefix[prefix.size() - 1] == ':' + && isValidSchemeName(prefix.substr(0, prefix.size() - 1)); +} +bool isAllowedURI(std::string_view uri, const Strings & allowedUris) +{ /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; + access to https://github.com. */ + for (auto & prefix : allowedUris) { + if (uri == prefix + // Allow access to subdirectories of the prefix. + || (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && ( + // Allow access to subdirectories of the prefix. + prefix[prefix.size() - 1] == '/' + || uri[prefix.size()] == '/' + + // Allow access to whole schemes + || isJustSchemePrefix(prefix) + ) + )) + return true; + } + + return false; +} + +void EvalState::checkURI(const std::string & uri) +{ + if (!evalSettings.restrictEval) return; + + if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ @@ -683,6 +591,26 @@ void PrimOp::check() } +std::ostream & operator<<(std::ostream & output, PrimOp & primOp) +{ + output << "primop " << primOp.name; + return output; +} + + +PrimOp * Value::primOpAppPrimOp() const +{ + Value * left = primOpApp.left; + while (left && !left->isPrimOp()) { + left = left->primOpApp.left; + } + + if (!left) + return nullptr; + return left->primOp; +} + + void Value::mkPrimOp(PrimOp * p) { p->check(); @@ -757,7 +685,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) // just for the current level of Env, not the whole chain. void printWithBindings(const SymbolTable & st, const Env & env) { - if (env.type == Env::HasWithAttrs) { + if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; Bindings::iterator j = env.values[0]->attrs->begin(); @@ -777,7 +705,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & if (se.up && env.up) { std::cout << "static: "; printStaticEnvBindings(st, se); - printWithBindings(st, env); + if (se.isWith) + printWithBindings(st, env); std::cout << std::endl; printEnvBindings(st, *se.up, *env.up, ++lvl); } else { @@ -789,7 +718,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; - printWithBindings(st, env); // probably nothing there for the top level. + if (se.isWith) + printWithBindings(st, env); // probably nothing there for the top level. std::cout << std::endl; } @@ -811,7 +741,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (env.up && se.up) { mapStaticEnvBindings(st, *se.up, *env.up, vm); - if (env.type == Env::HasWithAttrs) { + if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. Bindings::iterator j = env.values[0]->attrs->begin(); while (j != env.values[0]->attrs->end()) { @@ -844,7 +774,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & ? std::make_unique( *this, DebugTrace { - .pos = error->info().errPos ? error->info().errPos : static_cast>(positions[expr.getPos()]), + .pos = error->info().pos ? error->info().pos : positions[expr.getPos()], .expr = expr, .env = env, .hint = error->info().msg, @@ -854,12 +784,10 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & if (error) { - printError("%s\n\n", error->what()); + printError("%s\n", error->what()); if (trylevel > 0 && error->info().level != lvlInfo) printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); - - printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL); } auto se = getStaticEnv(expr); @@ -876,23 +804,23 @@ void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const { - e.addTrace(positions[pos], hintfmt(s, s2), frame); + e.addTrace(positions[pos], HintFmt(s, s2), frame); } +template static std::unique_ptr makeDebugTraceStacker( EvalState & state, Expr & expr, Env & env, - std::shared_ptr && pos, - const char * s, - const std::string & s2) + std::shared_ptr && pos, + const Args & ... formatArgs) { return std::make_unique(state, DebugTrace { .pos = std::move(pos), .expr = expr, .env = env, - .hint = hintfmt(s, s2), + .hint = HintFmt(formatArgs...), .isError = false }); } @@ -949,22 +877,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) if (!var.fromWith) return env->values[var.displ]; + // This early exit defeats the `maybeThunk` optimization for variables from `with`, + // The added complexity of handling this appears to be similarly in cost, or + // the cases where applicable were insignificant in the first place. + if (noEval) return nullptr; + + auto * fromWith = var.fromWith; while (1) { - if (env->type == Env::HasWithExpr) { - if (noEval) return 0; - Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, ""); - env->values[0] = v; - env->type = Env::HasWithAttrs; - } + forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression"); Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { if (countCalls) attrSelects[j->pos]++; return j->value; } - if (!env->prevWith) - error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); - for (size_t l = env->prevWith; l; --l, env = env->up) ; + if (!fromWith->parentWith) + error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); + for (size_t l = fromWith->prevWith; l; --l, env = env->up) ; + fromWith = fromWith->parentWith; } } @@ -1160,7 +1089,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) *this, *e, this->baseEnv, - e->getPos() ? static_cast>(positions[e->getPos()]) : nullptr, + e->getPos() ? std::make_shared(positions[e->getPos()]) : nullptr, "while evaluating the file '%1%':", resolvedPath.to_string()) : nullptr; @@ -1168,7 +1097,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) // computation. if (mustBeTrivial && !(dynamic_cast(e))) - error("file '%s' must be an attribute set", path).debugThrow(); + error("file '%s' must be an attribute set", path).debugThrow(); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); @@ -1199,7 +1128,11 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri Value v; e->eval(*this, env, v); if (v.type() != nBool) - error("value is %1% while a Boolean was expected", showType(v)).withFrame(env, *e).debugThrow(); + error( + "expected a Boolean but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).withFrame(env, *e).debugThrow(); return v.boolean; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); @@ -1213,7 +1146,11 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po try { e->eval(*this, env, v); if (v.type() != nAttrs) - error("value is %1% while a set was expected", showType(v)).withFrame(env, *e).debugThrow(); + error( + "expected a set but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withFrame(env, *e).debugThrow(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -1322,7 +1259,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); + state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ @@ -1348,6 +1285,19 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env2, + getPos() + ? std::make_shared(state.positions[getPos()]) + : nullptr, + "while evaluating a '%1%' expression", + "let" + ) + : nullptr; + body->eval(state, env2, v); } @@ -1360,6 +1310,15 @@ void ExprList::eval(EvalState & state, Env & env, Value & v) } +Value * ExprList::maybeThunk(EvalState & state, Env & env) +{ + if (elems.empty()) { + return &state.vEmptyList; + } + return Expr::maybeThunk(state, env); +} + + void ExprVar::eval(EvalState & state, Env & env, Value & v) { Value * v2 = state.lookupVar(&env, *this, false); @@ -1425,8 +1384,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) for (auto & attr : *vAttrs->attrs) allAttrNames.insert(state.symbols[attr.name]); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); - state.error("attribute '%1%' missing", state.symbols[name]) - .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); + state.error("attribute '%1%' missing", state.symbols[name]) + .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); } } vAttrs = j->value; @@ -1481,9 +1440,27 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } +namespace { +/** Increments a count on construction and decrements on destruction. + */ +class CallDepth { + size_t & count; +public: + CallDepth(size_t & count) : count(count) { + ++count; + } + ~CallDepth() { + --count; + } +}; +}; void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { + if (callDepth > evalSettings.maxCallDepth) + error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); + CallDepth _level(callDepth); + auto trace = evalSettings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; @@ -1539,13 +1516,13 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto j = args[0]->attrs->get(i.name); if (!j) { if (!i.def) { - error("function '%1%' called without required argument '%2%'", + error("function '%1%' called without required argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), symbols[i.name]) .atPos(lambda.pos) .withTrace(pos, "from call site") .withFrame(*fun.lambda.env, lambda) - .debugThrow(); + .debugThrow(); } env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { @@ -1565,14 +1542,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & for (auto & formal : lambda.formals->formals) formalNames.insert(symbols[formal.name]); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); - error("function '%1%' called with unexpected argument '%2%'", + error("function '%1%' called with unexpected argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), symbols[i.name]) .atPos(lambda.pos) .withTrace(pos, "from call site") .withSuggestions(suggestions) .withFrame(*fun.lambda.env, lambda) - .debugThrow(); + .debugThrow(); } abort(); // can't happen } @@ -1623,15 +1600,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & return; } else { /* We have all the arguments, so call the primop. */ - auto name = vCur.primOp->name; + auto * fn = vCur.primOp; nrPrimOpCalls++; - if (countCalls) primOpCalls[name]++; + if (countCalls) primOpCalls[fn->name]++; try { - vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur); + fn->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { - addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } @@ -1668,18 +1645,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & for (size_t i = 0; i < argsLeft; ++i) vArgs[argsDone + i] = args[i]; - auto name = primOp->primOp->name; + auto fn = primOp->primOp; nrPrimOpCalls++; - if (countCalls) primOpCalls[name]++; + if (countCalls) primOpCalls[fn->name]++; try { // TODO: // 1. Unify this and above code. Heavily redundant. // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur); + fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { - addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } @@ -1705,7 +1682,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } else - error("attempt to call something which is not a function but %1%", showType(vCur)).atPos(pos).debugThrow(); + error( + "attempt to call something which is not a function but %1%: %2%", + showType(vCur), + ValuePrinter(*this, vCur, errorPrintOptions)) + .atPos(pos) + .debugThrow(); } vRes = vCur; @@ -1714,6 +1696,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & void ExprCall::eval(EvalState & state, Env & env, Value & v) { + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env, + getPos() + ? std::make_shared(state.positions[getPos()]) + : nullptr, + "while calling a function" + ) + : nullptr; + Value vFun; fun->eval(state, env, vFun); @@ -1775,12 +1769,12 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { attrs.insert(*j); } else if (!i.def) { - error(R"(cannot evaluate a function that has an argument without a value ('%1%') + error(R"(cannot evaluate a function that has an argument without a value ('%1%') Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name]) - .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow(); + .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow(); } } } @@ -1793,9 +1787,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) { Env & env2(state.allocEnv(1)); env2.up = &env; - env2.prevWith = prevWith; - env2.type = Env::HasWithExpr; - env2.values[0] = (Value *) attrs; + env2.values[0] = attrs->maybeThunk(state, env); body->eval(state, env2, v); } @@ -1813,7 +1805,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { std::ostringstream out; cond->show(state.symbols, out); - state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); + state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -2004,24 +1996,24 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp->fpoint; } else - state.error("cannot add %1% to an integer", showType(*vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); + state.error("cannot add %1% to an integer", showType(*vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp->type() == nInt) { nf += vTmp->integer; } else if (vTmp->type() == nFloat) { nf += vTmp->fpoint; } else - state.error("cannot add %1% to a float", showType(*vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); + state.error("cannot add %1% to a float", showType(*vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nPath) { if (!first) { auto part = state.coerceToString(i_pos, *vTmp, context, "while evaluating a path segment", false, false); if (sSize <= 1 && !hasPrefix(*part, "/") && accessor != state.rootFS.get_ptr() && !part->empty()) - state.error( + state.error( "cannot append non-absolute path '%1%' to '%2%' (hint: change it to '/%1%')", (std::string) *part, SourcePath(ref(accessor)).to_string()) .atPos(i_pos) .withFrame(env, *this) - .debugThrow(); + .debugThrow(); sSize += part->size(); s.emplace_back(std::move(part)); } @@ -2041,7 +2033,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) - state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); + state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); v.mkPath({ref(accessor), CanonPath(str())}); } else v.mkStringMove(c_str(), context); @@ -2054,6 +2046,30 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) } +void ExprBlackHole::eval(EvalState & state, Env & env, Value & v) +{ + state.error("infinite recursion encountered") + .atPos(v.determinePos(noPos)) + .debugThrow(); +} + +// always force this to be separate, otherwise forceValue may inline it and take +// a massive perf hit +[[gnu::noinline]] +void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) +{ + if (!v.isBlackhole()) + return; + auto e = std::current_exception(); + try { + std::rethrow_exception(e); + } catch (InfiniteRecursionError & e) { + e.atPos(positions[pos]); + } catch (...) { + } +} + + void EvalState::forceValueDeep(Value & v) { std::set seen; @@ -2063,7 +2079,7 @@ void EvalState::forceValueDeep(Value & v) recurse = [&](Value & v) { if (!seen.insert(&v).second) return; - forceValue(v, [&]() { return v.determinePos(noPos); }); + forceValue(v, v.determinePos(noPos)); if (v.type() == nAttrs) { for (auto & i : *v.attrs) @@ -2096,12 +2112,18 @@ NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCt try { forceValue(v, pos); if (v.type() != nInt) - error("value is %1% while an integer was expected", showType(v)).debugThrow(); + error( + "expected an integer but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.integer; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + + return v.integer; } @@ -2112,7 +2134,11 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err if (v.type() == nInt) return v.integer; else if (v.type() != nFloat) - error("value is %1% while a float was expected", showType(v)).debugThrow(); + error( + "expected a float but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.fpoint; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); @@ -2126,12 +2152,18 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx try { forceValue(v, pos); if (v.type() != nBool) - error("value is %1% while a Boolean was expected", showType(v)).debugThrow(); + error( + "expected a Boolean but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.boolean; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + + return v.boolean; } @@ -2146,7 +2178,11 @@ void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view erro try { forceValue(v, pos); if (v.type() != nFunction && !isFunctor(v)) - error("value is %1% while a function was expected", showType(v)).debugThrow(); + error( + "expected a function but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2159,7 +2195,11 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string try { forceValue(v, pos); if (v.type() != nString) - error("value is %1% while a string was expected", showType(v)).debugThrow(); + error( + "expected a string but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); @@ -2188,7 +2228,7 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s { auto s = forceString(v, pos, errorCtx); if (v.context()) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2248,16 +2288,20 @@ BackedStringView EvalState::coerceToString( return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) { - error("cannot coerce %1% to a string", showType(v)) + error( + "cannot coerce %1% to a string: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ) .withTrace(pos, errorCtx) - .debugThrow(); + .debugThrow(); } return coerceToString(pos, *i->value, context, errorCtx, coerceMore, copyToStore); } if (v.type() == nExternal) { try { - return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore); + return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore); } catch (Error & e) { e.addTrace(nullptr, errorCtx); throw; @@ -2293,23 +2337,26 @@ BackedStringView EvalState::coerceToString( } } - error("cannot coerce %1% to a string", showType(v)) + error("cannot coerce %1% to a string: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ) .withTrace(pos, errorCtx) - .debugThrow(); + .debugThrow(); } StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) { if (nix::isDerivation(path.path.abs())) - error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); + error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); auto i = srcToStore.find(path); auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); + auto dstPath = fetchToStore(*store, path.resolveSymlinks(), path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); @@ -2363,7 +2410,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon auto path = coerceToString(pos, v, context, errorCtx, false, false).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); + error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); } @@ -2373,18 +2420,18 @@ std::pair EvalState::coerceToSingleDerivedP auto s = forceString(v, context, pos, errorCtx); auto csize = context.size(); if (csize != 1) - error( + error( "string '%s' has %d entries in its context. It should only have exactly one entry", s, csize) - .withTrace(pos, errorCtx).debugThrow(); + .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { return std::move(o); }, [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { - error( + error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", - s).withTrace(pos, errorCtx).debugThrow(); + s).withTrace(pos, errorCtx).debugThrow(); }, [&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); @@ -2407,16 +2454,16 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & error message. */ std::visit(overloaded { [&](const SingleDerivedPath::Opaque & o) { - error( + error( "path string '%s' has context with the different path '%s'", s, sExpected) - .withTrace(pos, errorCtx).debugThrow(); + .withTrace(pos, errorCtx).debugThrow(); }, [&](const SingleDerivedPath::Built & b) { - error( + error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", s, b.output, b.drvPath->to_string(*store), sExpected) - .withTrace(pos, errorCtx).debugThrow(); + .withTrace(pos, errorCtx).debugThrow(); } }, derivedPath.raw()); } @@ -2451,7 +2498,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return v1.string_view().compare(v2.string_view()) == 0; + return strcmp(v1.c_str(), v2.c_str()) == 0; case nPath: return @@ -2501,7 +2548,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nThunk: // Must not be left by forceValue default: - error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); + error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); } } @@ -2652,11 +2699,185 @@ void EvalState::printStatistics() } -std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const +SourcePath resolveExprPath(SourcePath path) { - throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string", showType()) - }); + unsigned int followCount = 0, maxFollow = 1024; + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + while (!path.path.isRoot()) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + auto p = path.parent().resolveSymlinks() / path.baseName(); + if (p.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) + return path / "default.nix"; + + return path; +} + + +Expr * EvalState::parseExprFromFile(const SourcePath & path) +{ + return parseExprFromFile(path, staticBaseEnv); +} + + +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) +{ + auto buffer = path.resolveSymlinks().readFile(); + // readFile hopefully have left some extra space for terminators + buffer.append("\0\0", 2); + return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); +} + + +Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) +{ + auto s = make_ref(std::move(s_)); + s->append("\0\0", 2); + return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); +} + + +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) +{ + return parseExprFromString(std::move(s), basePath, staticBaseEnv); +} + + +Expr * EvalState::parseStdin() +{ + //Activity act(*logger, lvlTalkative, "parsing standard input"); + auto buffer = drainFD(0); + // drainFD should have left some extra space for terminators + buffer.append("\0\0", 2); + auto s = make_ref(std::move(buffer)); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv); +} + + +SourcePath EvalState::findFile(const std::string_view path) +{ + return findFile(searchPath, path); +} + + +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) +{ + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + auto res = r / CanonPath(suffix); + if (res.pathExists()) return res; + } + + if (hasPrefix(path, "nix/")) + return {corepkgsFS, CanonPath(path.substr(3))}; + + error( + evalSettings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path + ).atPos(pos).debugThrow(); +} + + +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) +{ + auto & value = value0.s; + auto i = searchPathResolved.find(value); + if (i != searchPathResolved.end()) return i->second; + + std::optional res; + + if (EvalSettings::isPseudoUrl(value)) { + try { + auto accessor = fetchers::downloadTarball( + EvalSettings::resolvePseudoUrl(value)).accessor; + registerAccessor(accessor); + res.emplace(SourcePath(accessor)); + } catch (Error & e) { + logWarning({ + .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) + }); + } + } + + else if (hasPrefix(value, "flake:")) { + experimentalFeatureSettings.require(Xp::Flakes); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); + auto [accessor, _] = flakeRef.resolve(store).lazyFetch(store); + res.emplace(SourcePath(accessor)); + } + + else { + auto path = rootPath(value); + + /* Allow access to paths in the search path. */ + if (initAccessControl) { + allowPath(path.path.abs()); + if (store->isInStore(path.path.abs())) { + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); + for (auto & p : closure) + allowPath(p); + } catch (InvalidPath &) { } + } + } + + if (path.pathExists()) + res.emplace(path); + else { + logWarning({ + .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value) + }); + } + } + + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + + searchPathResolved.emplace(value, res); + return res; +} + + +Expr * EvalState::parse( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + std::shared_ptr & staticEnv) +{ + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols); + + result->bindVars(*this, staticEnv); + + return result; +} + + +std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const +{ + state.error( + "cannot coerce %1% to a string: %2%", showType(), *this + ).atPos(pos).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 04111034222..76b154fa096 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -2,6 +2,7 @@ ///@file #include "attr-set.hh" +#include "eval-error.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" @@ -86,6 +87,8 @@ struct PrimOp void check(); }; +std::ostream & operator<<(std::ostream & output, PrimOp & primOp); + /** * Info about a constant */ @@ -118,11 +121,6 @@ struct Constant struct Env { Env * up; - /** - * Number of of levels up to next `with` environment - */ - unsigned short prevWith:14; - enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -134,7 +132,7 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati void copyContext(const Value & v, NixStringContext & context); -std::string printValue(const EvalState & state, const Value & v); +std::string printValue(EvalState & state, Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); @@ -149,52 +147,13 @@ struct RegexCache; std::shared_ptr makeRegexCache(); struct DebugTrace { - std::shared_ptr pos; + std::shared_ptr pos; const Expr & expr; const Env & env; - hintformat hint; + HintFmt hint; bool isError; }; -void debugError(Error * e, Env & env, Expr & expr); - -class ErrorBuilder -{ - private: - EvalState & state; - ErrorInfo info; - - ErrorBuilder(EvalState & s, ErrorInfo && i): state(s), info(i) { } - - public: - template - [[nodiscard, gnu::noinline]] - static ErrorBuilder * create(EvalState & s, const Args & ... args) - { - return new ErrorBuilder(s, ErrorInfo { .msg = hintfmt(args...) }); - } - - [[nodiscard, gnu::noinline]] - ErrorBuilder & atPos(PosIdx pos); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withTrace(PosIdx pos, const std::string_view text); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withSuggestions(Suggestions & s); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withFrame(const Env & e, const Expr & ex); - - template - [[gnu::noinline, gnu::noreturn]] - void debugThrow(); -}; - - class EvalState : public std::enable_shared_from_this { public: @@ -212,6 +171,8 @@ public: sPrefix, sOutputSpecified; + const Expr::AstSymbols exprSymbols; + /** * If set, force copying files to the Nix store even if they * already exist there. @@ -220,6 +181,11 @@ public: Bindings emptyBindings; + /** + * Empty list constant. + */ + Value vEmptyList; + /** * The accessor for the root filesystem. */ @@ -276,39 +242,11 @@ public: void runDebugRepl(const Error * error, const Env & env, const Expr & expr); - template - [[gnu::noinline, gnu::noreturn]] - void debugThrowLastTrace(E && error) - { - debugThrow(error, nullptr, nullptr); - } - - template - [[gnu::noinline, gnu::noreturn]] - void debugThrow(E && error, const Env * env, const Expr * expr) - { - if (debugRepl && ((env && expr) || !debugTraces.empty())) { - if (!env || !expr) { - const DebugTrace & last = debugTraces.front(); - env = &last.env; - expr = &last.expr; - } - runDebugRepl(&error, *env, *expr); - } - - throw std::move(error); - } - - // This is dangerous, but gets in line with the idea that error creation and - // throwing should not allocate on the stack of hot functions. - // as long as errors are immediately thrown, it works. - ErrorBuilder * errorBuilder; - - template + template [[nodiscard, gnu::noinline]] - ErrorBuilder & error(const Args & ... args) { - errorBuilder = ErrorBuilder::create(*this, args...); - return *errorBuilder; + EvalErrorBuilder & error(const Args & ... args) { + // `EvalErrorBuilder::debugThrow` performs the corresponding `delete`. + return *new EvalErrorBuilder(*this, args...); } private: @@ -390,6 +328,11 @@ public: corresponding input accessor. */ std::string decodePaths(std::string_view s); + /** + * Variant which accepts relative paths too. + */ + SourcePath rootPath(PathView path); + /** * Allow access to a path. */ @@ -482,8 +425,7 @@ public: */ inline void forceValue(Value & v, const PosIdx pos); - template - inline void forceValue(Value & v, Callable getPos); + void tryFixupBlackHolePos(Value & v, PosIdx pos); /** * Force a value, then recursively force list elements and @@ -644,6 +586,11 @@ private: const SourcePath & basePath, std::shared_ptr & staticEnv); + /** + * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack. + */ + size_t callDepth = 0; + public: /** @@ -853,21 +800,10 @@ std::string showType(const Value & v); */ SourcePath resolveExprPath(SourcePath path); -struct InvalidPathError : EvalError -{ - Path path; - InvalidPathError(const Path & path); -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~InvalidPathError() throw () { }; -#endif -}; - -template -void ErrorBuilder::debugThrow() -{ - // NOTE: We always use the -LastTrace version as we push the new trace in withFrame() - state.debugThrowLastTrace(ErrorType(info)); -} +/** + * Whether a URI is allowed, assuming restrictEval is enabled + */ +bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0fefe394f8f..e377786033d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -10,6 +10,7 @@ #include "finally.hh" #include "fetch-settings.hh" #include "value-to-json.hh" +#include "patching-input-accessor.hh" namespace nix { @@ -87,8 +88,8 @@ static FlakeInput parseFlakeInput( input.patchFiles.emplace_back(flakePath.parent().path.makeRelative(elem->path().path)); } else - throw TypeError("flake input attribute '%s' is %s while a string or path is expected", - state.symbols[attr.name], showType(*elem)); + state.error("flake input attribute '%s' is %s while a string or path is expected", + state.symbols[attr.name], showType(*elem)).debugThrow(); } } else { // Allow selecting a subset of enum values @@ -110,15 +111,15 @@ static FlakeInput parseFlakeInput( NixStringContext emptyContext = {}; attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); } else - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + state.error("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); } #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( state.positions[attr.pos], - hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); + HintFmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); throw; } } @@ -127,7 +128,7 @@ static FlakeInput parseFlakeInput( try { input.ref = FlakeRef::fromAttrs(attrs); } catch (Error & e) { - e.addTrace(state.positions[pos], hintfmt("while evaluating flake input")); + e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; } else { @@ -177,7 +178,7 @@ static Flake readFlake( const InputPath & lockRootPath) { CanonPath flakeDir(resolvedRef.subdir); - auto flakePath = rootDir + flakeDir + "flake.nix"; + auto flakePath = rootDir / flakeDir / "flake.nix"; Value vInfo; state.evalFile(flakePath, vInfo, true); @@ -247,15 +248,15 @@ static Flake readFlake( std::vector ss; for (auto elem : setting.value->listItems()) { if (elem->type() != nString) - throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", - state.symbols[setting.name], showType(*setting.value)); + state.error("list element in flake configuration setting '%s' is %s while a string is expected", + state.symbols[setting.name], showType(*setting.value)).debugThrow(); ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, "")); } flake.config.settings.emplace(state.symbols[setting.name], ss); } else - throw TypeError("flake configuration setting '%s' is %s", - state.symbols[setting.name], showType(*setting.value)); + state.error("flake configuration setting '%s' is %s", + state.symbols[setting.name], showType(*setting.value)).debugThrow(); } } @@ -712,7 +713,7 @@ LockedFlake lockFlake( if (lockFlags.outputLockFilePath) { if (lockFlags.commitLockFile) throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); - writeFile(lockFlags.outputLockFilePath->abs(), newLockFileS); + writeFile(*lockFlags.outputLockFilePath, newLockFileS); } else { bool lockFileExists = flake->lockFilePath().pathExists(); @@ -925,11 +926,11 @@ static void prim_flakeRefToString( attrs.emplace(state.symbols[attr.name], std::string(attr.value->string_view())); } else { - state.error( + state.error( "flake reference attribute sets may only contain integers, Booleans, " "and strings, but attribute '%s' is %s", state.symbols[attr.name], - showType(*attr.value)).debugThrow(); + showType(*attr.value)).debugThrow(); } } auto flakeRef = FlakeRef::fromAttrs(attrs); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c11fcd0b47b..d45382a6340 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -97,7 +97,7 @@ struct Flake SourcePath lockFilePath() { - return path.parent() + "flake.lock"; + return path.parent() / "flake.lock"; } }; @@ -178,7 +178,7 @@ struct LockFlags /** * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ - std::optional outputLockFilePath; + std::optional outputLockFilePath; /** * Flake inputs to be overridden. diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 0ccee7dabe9..78537d181cf 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -207,7 +207,7 @@ std::optional> parseFlakeIdRef( static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" - + "(?:#(" + queryRegex + "))?", + + "(?:#(" + fragmentRegex + "))?", std::regex::ECMAScript); if (std::regex_match(url, match, flakeRegex)) { diff --git a/src/libexpr/flake/url-name.cc b/src/libexpr/flake/url-name.cc new file mode 100644 index 00000000000..753f197d555 --- /dev/null +++ b/src/libexpr/flake/url-name.cc @@ -0,0 +1,48 @@ +#include "url-name.hh" +#include +#include + +namespace nix { + +static const std::string attributeNamePattern("[a-zA-Z0-9_-]+"); +static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?"); +static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+"); +static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); +static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?"); +static const std::regex gitProviderRegex("github|gitlab|sourcehut"); +static const std::regex gitSchemeRegex("git($|\\+.*)"); +static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)"); + +std::optional getNameFromURL(const ParsedURL & url) +{ + std::smatch match; + + /* If there is a dir= argument, use its value */ + if (url.query.count("dir") > 0) + return url.query.at("dir"); + + /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ + if (std::regex_match(url.fragment, match, lastAttributeRegex)) + return match.str(1); + + /* If this is a github/gitlab/sourcehut flake, use the repo name */ + if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) + return match.str(1); + + /* If it is a regular git flake, use the directory name */ + if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If everything failed but there is a non-default fragment, use it in full */ + if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex)) + return url.fragment; + + /* If there is no fragment, take the last element of the path */ + if (std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If even that didn't work, the URL does not contain enough info to determine a useful name */ + return {}; +} + +} diff --git a/src/libexpr/flake/url-name.hh b/src/libexpr/flake/url-name.hh new file mode 100644 index 00000000000..6f32754d268 --- /dev/null +++ b/src/libexpr/flake/url-name.hh @@ -0,0 +1,20 @@ +#include "url.hh" +#include "url-parts.hh" +#include "util.hh" +#include "split.hh" + +namespace nix { + +/** + * Try to extract a reasonably unique and meaningful, human-readable + * name of a flake output from a parsed URL. + * When nullopt is returned, the callsite should use information available + * to it outside of the URL to determine a useful name. + * This is a heuristic approach intended for user interfaces. + * @return nullopt if the extracted name is not useful to identify a + * flake output, for example because it is empty or "default". + * Otherwise returns the extracted name. + */ +std::optional getNameFromURL(const ParsedURL & url); + +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d4e946d8104..e9ed1ef0891 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -11,13 +11,13 @@ namespace nix { -DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs) +PackageInfo::PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs) : state(&state), attrs(attrs), attrPath(std::move(attrPath)) { } -DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) +PackageInfo::PackageInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); @@ -45,18 +45,18 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat } -std::string DrvInfo::queryName() const +std::string PackageInfo::queryName() const { if (name == "" && attrs) { auto i = attrs->find(state->sName); - if (i == attrs->end()) throw TypeError("derivation name missing"); + if (i == attrs->end()) state->error("derivation name missing").debugThrow(); name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation"); } return name; } -std::string DrvInfo::querySystem() const +std::string PackageInfo::querySystem() const { if (system == "" && attrs) { auto i = attrs->find(state->sSystem); @@ -66,7 +66,7 @@ std::string DrvInfo::querySystem() const } -std::optional DrvInfo::queryDrvPath() const +std::optional PackageInfo::queryDrvPath() const { if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); @@ -80,7 +80,7 @@ std::optional DrvInfo::queryDrvPath() const } -StorePath DrvInfo::requireDrvPath() const +StorePath PackageInfo::requireDrvPath() const { if (auto drvPath = queryDrvPath()) return *drvPath; @@ -88,7 +88,7 @@ StorePath DrvInfo::requireDrvPath() const } -StorePath DrvInfo::queryOutPath() const +StorePath PackageInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); @@ -102,7 +102,7 @@ StorePath DrvInfo::queryOutPath() const } -DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) +PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ @@ -164,7 +164,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall } -std::string DrvInfo::queryOutputName() const +std::string PackageInfo::queryOutputName() const { if (outputName == "" && attrs) { Bindings::iterator i = attrs->find(state->sOutputName); @@ -174,7 +174,7 @@ std::string DrvInfo::queryOutputName() const } -Bindings * DrvInfo::getMeta() +Bindings * PackageInfo::getMeta() { if (meta) return meta; if (!attrs) return 0; @@ -186,7 +186,7 @@ Bindings * DrvInfo::getMeta() } -StringSet DrvInfo::queryMetaNames() +StringSet PackageInfo::queryMetaNames() { StringSet res; if (!getMeta()) return res; @@ -196,9 +196,9 @@ StringSet DrvInfo::queryMetaNames() } -bool DrvInfo::checkMeta(Value & v) +bool PackageInfo::checkMeta(Value & v) { - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); if (v.type() == nList) { for (auto elem : v.listItems()) if (!checkMeta(*elem)) return false; @@ -216,7 +216,7 @@ bool DrvInfo::checkMeta(Value & v) } -Value * DrvInfo::queryMeta(const std::string & name) +Value * PackageInfo::queryMeta(const std::string & name) { if (!getMeta()) return 0; Bindings::iterator a = meta->find(state->symbols.create(name)); @@ -225,7 +225,7 @@ Value * DrvInfo::queryMeta(const std::string & name) } -std::string DrvInfo::queryMetaString(const std::string & name) +std::string PackageInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; @@ -233,7 +233,7 @@ std::string DrvInfo::queryMetaString(const std::string & name) } -NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) +NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; @@ -247,7 +247,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) return def; } -NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) +NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; @@ -262,7 +262,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) } -bool DrvInfo::queryMetaBool(const std::string & name, bool def) +bool PackageInfo::queryMetaBool(const std::string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; @@ -277,7 +277,7 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def) } -void DrvInfo::setMeta(const std::string & name, Value * v) +void PackageInfo::setMeta(const std::string & name, Value * v) { getMeta(); auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0)); @@ -300,18 +300,18 @@ typedef std::set Done; The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ static bool getDerivation(EvalState & state, Value & v, - const std::string & attrPath, DrvInfos & drvs, Done & done, + const std::string & attrPath, PackageInfos & drvs, Done & done, bool ignoreAssertionFailures) { try { - state.forceValue(v, [&]() { return v.determinePos(noPos); }); + state.forceValue(v, v.determinePos(noPos)); if (!state.isDerivation(v)) return true; /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ if (!done.insert(v.attrs).second) return false; - DrvInfo drv(state, attrPath, v.attrs); + PackageInfo drv(state, attrPath, v.attrs); drv.queryName(); @@ -326,11 +326,11 @@ static bool getDerivation(EvalState & state, Value & v, } -std::optional getDerivation(EvalState & state, Value & v, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; - DrvInfos drvs; + PackageInfos drvs; getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); if (drvs.size() != 1) return {}; return std::move(drvs.front()); @@ -348,7 +348,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); static void getDerivations(EvalState & state, Value & vIn, const std::string & pathPrefix, Bindings & autoArgs, - DrvInfos & drvs, Done & done, + PackageInfos & drvs, Done & done, bool ignoreAssertionFailures) { Value v; @@ -396,12 +396,13 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); + else + state.error("expression does not evaluate to a derivation (or a set or list of those)").debugThrow(); } void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) + Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures) { Done done; getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 584d64ac1d1..e8c1190f7dc 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -10,8 +10,10 @@ namespace nix { - -struct DrvInfo +/** + * A "parsed" package attribute set. + */ +struct PackageInfo { public: typedef std::map> Outputs; @@ -43,9 +45,9 @@ public: */ std::string attrPath; - DrvInfo(EvalState & state) : state(&state) { }; - DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); - DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); + PackageInfo(EvalState & state) : state(&state) { }; + PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs); + PackageInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); std::string queryName() const; std::string querySystem() const; @@ -82,21 +84,21 @@ public: #if HAVE_BOEHMGC -typedef std::list> DrvInfos; +typedef std::list> PackageInfos; #else -typedef std::list DrvInfos; +typedef std::list PackageInfos; #endif /** - * If value `v` denotes a derivation, return a DrvInfo object + * If value `v` denotes a derivation, return a PackageInfo object * describing it. Otherwise return nothing. */ -std::optional getDerivation(EvalState & state, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, + Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures); diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 99a475ff987..2d12c47c536 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -1,4 +1,6 @@ #include "json-to-value.hh" +#include "value.hh" +#include "eval.hh" #include #include @@ -159,7 +161,7 @@ class JSONSax : nlohmann::json_sax { } bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { - throw JSONParseError(ex.what()); + throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 3b8ec000ff0..3c8fa5cc00a 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -1,13 +1,16 @@ #pragma once ///@file -#include "eval.hh" +#include "error.hh" #include namespace nix { -MakeError(JSONParseError, EvalError); +class EvalState; +struct Value; + +MakeError(JSONParseError, Error); void parseJSON(EvalState & state, const std::string_view & s, Value & v); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a3a8608d9a7..380048c77e2 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -1,4 +1,5 @@ %option reentrant bison-bridge bison-locations +%option align %option noyywrap %option never-interactive %option stack @@ -28,15 +29,7 @@ using namespace nix; namespace nix { -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylloc, data) - -// backup to recover from yyless(0) -thread_local YYLTYPE prev_yylloc; +#define CUR_POS state->at(*yylloc) static void initLoc(YYLTYPE * loc) { @@ -46,7 +39,7 @@ static void initLoc(YYLTYPE * loc) static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) { - prev_yylloc = *loc; + loc->stash(); loc->first_line = loc->last_line; loc->first_column = loc->last_column; @@ -132,7 +125,7 @@ else { return ELSE; } assert { return ASSERT; } with { return WITH; } let { return LET; } -in { return IN; } +in { return IN_KW; } rec { return REC; } inherit { return INHERIT; } or { return OR_KW; } @@ -153,21 +146,21 @@ or { return OR_KW; } try { yylval->n = boost::lexical_cast(yytext); } catch (const boost::bad_lexical_cast &) { - throw ParseError({ - .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("invalid integer '%1%'", yytext), + .pos = state->positions[CUR_POS], }); } - return INT; + return INT_LIT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError({ - .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("invalid float '%1%'", yytext), + .pos = state->positions[CUR_POS], }); - return FLOAT; + return FLOAT_LIT; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -188,7 +181,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->str = unescapeStr(data->symbols, yytext, yyleng); + yylval->str = unescapeStr(state->symbols, yytext, yyleng); return STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -216,7 +209,7 @@ or { return OR_KW; } return IND_STR; } \'\'\\{ANY} { - yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2); return IND_STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -230,7 +223,7 @@ or { return OR_KW; } {HPATH_START}\$\{ { PUSH_STATE(PATH_START); yyless(0); - *yylloc = prev_yylloc; + yylloc->unstash(); } {PATH_SEG} { @@ -286,15 +279,15 @@ or { return OR_KW; } context (it may be ')', ';', or something of that sort) */ POP_STATE(); yyless(0); - *yylloc = prev_yylloc; + yylloc->unstash(); return PATH_END; } {ANY} | <> { - throw ParseError({ - .msg = hintfmt("path has a trailing slash"), - .errPos = data->state.positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("path has a trailing slash"), + .pos = state->positions[CUR_POS], }); } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed6bc761a03..0c3e36750e8 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,9 +16,9 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) ifdef HOST_LINUX - libexpr_LDFLAGS += -ldl + libexpr_LDFLAGS += -ldl endif # The dependency on libgc must be propagated (i.e. meaning that diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index e76ce850513..f5e4d45acd5 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -9,57 +9,9 @@ namespace nix { -struct PosAdapter : AbstractPos -{ - Pos::Origin origin; - - PosAdapter(Pos::Origin origin) - : origin(std::move(origin)) - { - } +unsigned long Expr::nrExprs = 0; - std::optional getSource() const override - { - return std::visit(overloaded { - [](const Pos::none_tag &) -> std::optional { - return std::nullopt; - }, - [](const Pos::Stdin & s) -> std::optional { - // Get rid of the null terminators added by the parser. - return std::string(s.source->c_str()); - }, - [](const Pos::String & s) -> std::optional { - // Get rid of the null terminators added by the parser. - return std::string(s.source->c_str()); - }, - [](const SourcePath & path) -> std::optional { - try { - return path.readFile(); - } catch (Error &) { - return std::nullopt; - } - } - }, origin); - } - - void print(std::ostream & out) const override - { - std::visit(overloaded { - [&](const Pos::none_tag &) { out << "«none»"; }, - [&](const Pos::Stdin &) { out << "«stdin»"; }, - [&](const Pos::String & s) { out << "«string»"; }, - [&](const SourcePath & path) { out << path; } - }, origin); - } -}; - -Pos::operator std::shared_ptr() const -{ - auto pos = std::make_shared(origin); - pos->line = line; - pos->column = column; - return pos; -} +ExprBlackHole eBlackHole; // FIXME: remove, because *symbols* are abstract and do not have a single // textual representation; see printIdentifier() @@ -266,17 +218,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const } -std::ostream & operator << (std::ostream & str, const Pos & pos) -{ - if (auto pos2 = (std::shared_ptr) pos) { - str << *pos2; - } else - str << "undefined position"; - - return str; -} - - std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) { std::ostringstream out; @@ -331,6 +272,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); + fromWith = nullptr; + /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; @@ -342,7 +285,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & } else { auto i = curEnv->find(name); if (i != curEnv->vars.end()) { - fromWith = false; this->level = level; displ = i->second; return; @@ -354,11 +296,12 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ if (withLevel == -1) - throw UndefinedVarError({ - .msg = hintfmt("undefined variable '%1%'", es.symbols[name]), - .errPos = es.positions[pos] - }); - fromWith = true; + es.error( + "undefined variable '%1%'", + es.symbols[name] + ).atPos(pos).debugThrow(); + for (auto * e = env.get(); e && !fromWith; e = e->up) + fromWith = e->isWith; this->level = withLevel; } @@ -391,7 +334,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr es.exprEnvs.insert(std::make_pair(this, env)); if (recursive) { - auto newEnv = std::make_shared(false, env.get(), recursive ? attrs.size() : 0); + auto newEnv = std::make_shared(nullptr, env.get(), recursive ? attrs.size() : 0); Displacement displ = 0; for (auto & i : attrs) @@ -433,7 +376,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr es.exprEnvs.insert(std::make_pair(this, env)); auto newEnv = std::make_shared( - false, env.get(), + nullptr, env.get(), (hasFormals() ? formals->formals.size() : 0) + (!arg ? 0 : 1)); @@ -466,10 +409,7 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr & void ExprLet::bindVars(EvalState & es, const std::shared_ptr & env) { - if (es.debugRepl) - es.exprEnvs.insert(std::make_pair(this, env)); - - auto newEnv = std::make_shared(false, env.get(), attrs->attrs.size()); + auto newEnv = std::make_shared(nullptr, env.get(), attrs->attrs.size()); Displacement displ = 0; for (auto & i : attrs->attrs) @@ -480,6 +420,9 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr & for (auto & i : attrs->attrs) i.second.e->bindVars(es, i.second.inherited ? env : newEnv); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, newEnv)); + body->bindVars(es, newEnv); } @@ -488,6 +431,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); + parentWith = nullptr; + for (auto * e = env.get(); e && !parentWith; e = e->up) + parentWith = e->isWith; + /* Does this `with' have an enclosing `with'? If so, record its level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ @@ -500,11 +447,8 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & break; } - if (es.debugRepl) - es.exprEnvs.insert(std::make_pair(this, env)); - attrs->bindVars(es, env); - auto newEnv = std::make_shared(true, env.get()); + auto newEnv = std::make_shared(this, env.get()); body->bindVars(es, newEnv); } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e37058bf41d..fea2f08410a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -8,129 +8,18 @@ #include "symbol-table.hh" #include "error.hh" #include "chunked-vector.hh" +#include "position.hh" +#include "eval-error.hh" +#include "pos-idx.hh" +#include "pos-table.hh" namespace nix { -MakeError(EvalError, Error); -MakeError(ParseError, Error); -MakeError(AssertionError, EvalError); -MakeError(ThrownError, AssertionError); -MakeError(Abort, EvalError); -MakeError(TypeError, EvalError); -MakeError(UndefinedVarError, Error); -MakeError(MissingArgumentError, EvalError); - -/** - * Position objects. - */ -struct Pos -{ - uint32_t line; - uint32_t column; - - struct none_tag { }; - struct Stdin { ref source; }; - struct String { ref source; }; - - typedef std::variant Origin; - - Origin origin; - - explicit operator bool() const { return line > 0; } - - operator std::shared_ptr() const; -}; - -class PosIdx { - friend class PosTable; - -private: - uint32_t id; - - explicit PosIdx(uint32_t id): id(id) {} - -public: - PosIdx() : id(0) {} - - explicit operator bool() const { return id > 0; } - - bool operator <(const PosIdx other) const { return id < other.id; } - - bool operator ==(const PosIdx other) const { return id == other.id; } - - bool operator !=(const PosIdx other) const { return id != other.id; } -}; - -class PosTable -{ -public: - class Origin { - friend PosTable; - private: - // must always be invalid by default, add() replaces this with the actual value. - // subsequent add() calls use this index as a token to quickly check whether the - // current origins.back() can be reused or not. - mutable uint32_t idx = std::numeric_limits::max(); - - // Used for searching in PosTable::[]. - explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {} - - public: - const Pos::Origin origin; - - Origin(Pos::Origin origin): origin(origin) {} - }; - - struct Offset { - uint32_t line, column; - }; - -private: - std::vector origins; - ChunkedVector offsets; - -public: - PosTable(): offsets(1024) - { - origins.reserve(1024); - } - - PosIdx add(const Origin & origin, uint32_t line, uint32_t column) - { - const auto idx = offsets.add({line, column}).second; - if (origins.empty() || origins.back().idx != origin.idx) { - origin.idx = idx; - origins.push_back(origin); - } - return PosIdx(idx + 1); - } - - Pos operator[](PosIdx p) const - { - if (p.id == 0 || p.id > offsets.size()) - return {}; - const auto idx = p.id - 1; - /* we want the last key <= idx, so we'll take prev(first key > idx). - this is guaranteed to never rewind origin.begin because the first - key is always 0. */ - const auto pastOrigin = std::upper_bound( - origins.begin(), origins.end(), Origin(idx), - [] (const auto & a, const auto & b) { return a.idx < b.idx; }); - const auto origin = *std::prev(pastOrigin); - const auto offset = offsets[idx]; - return {offset.line, offset.column, origin.origin}; - } -}; - -inline PosIdx noPos = {}; - -std::ostream & operator << (std::ostream & str, const Pos & pos); - - struct Env; struct Value; class EvalState; +struct ExprWith; struct StaticEnv; @@ -154,6 +43,11 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + struct AstSymbols { + Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body; + }; + + static unsigned long nrExprs; Expr() { nrExprs++; @@ -219,8 +113,11 @@ struct ExprVar : Expr Symbol name; /* Whether the variable comes from an environment (e.g. a rec, let - or function argument) or from a "with". */ - bool fromWith; + or function argument) or from a "with". + + `nullptr`: Not from a `with`. + Valid pointer: the nearest, innermost `with` expression to query first. */ + ExprWith * fromWith; /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the @@ -292,6 +189,7 @@ struct ExprList : Expr std::vector elems; ExprList() { }; COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env) override; PosIdx getPos() const override { @@ -378,6 +276,7 @@ struct ExprWith : Expr PosIdx pos; Expr * attrs, * body; size_t prevWith; + ExprWith * parentWith; ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -455,20 +354,30 @@ struct ExprPos : Expr COMMON_METHODS }; +/* only used to mark thunks as black holes. */ +struct ExprBlackHole : Expr +{ + void show(const SymbolTable & symbols, std::ostream & str) const override {} + void eval(EvalState & state, Env & env, Value & v) override; + void bindVars(EvalState & es, const std::shared_ptr & env) override {} +}; + +extern ExprBlackHole eBlackHole; + /* Static environments are used to map variable names onto (level, displacement) pairs used to obtain the value of the variable at runtime. */ struct StaticEnv { - bool isWith; + ExprWith * isWith; const StaticEnv * up; // Note: these must be in sorted order. typedef std::vector> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { vars.reserve(expectedSize); }; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh new file mode 100644 index 00000000000..87aeaeef52a --- /dev/null +++ b/src/libexpr/parser-state.hh @@ -0,0 +1,271 @@ +#pragma once +///@file + +#include "eval.hh" + +namespace nix { + +/** + * @note Storing a C-style `char *` and `size_t` allows us to avoid + * having to define the special members that using string_view here + * would implicitly delete. + */ +struct StringToken +{ + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const { return {p, l}; } +}; + +struct ParserLocation +{ + int first_line, first_column; + int last_line, last_column; + + // backup to recover from yyless(0) + int stashed_first_line, stashed_first_column; + int stashed_last_line, stashed_last_column; + + void stash() { + stashed_first_line = first_line; + stashed_first_column = first_column; + stashed_last_line = last_line; + stashed_last_column = last_column; + } + + void unstash() { + first_line = stashed_first_line; + first_column = stashed_first_column; + last_line = stashed_last_line; + last_column = stashed_last_column; + } +}; + +struct ParserState +{ + SymbolTable & symbols; + PosTable & positions; + Expr * result; + SourcePath basePath; + PosTable::Origin origin; + const ref rootFS; + const Expr::AstSymbols & s; + + void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); + void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); + Expr * stripIndentation(const PosIdx pos, + std::vector>> && es); + PosIdx at(const ParserLocation & loc); +}; + +inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError({ + .msg = HintFmt("attribute '%1%' already defined at %2%", + showAttrPath(symbols, attrPath), positions[prevPos]), + .pos = positions[pos] + }); +} + +inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError({ + .msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]), + .pos = positions[pos] + }); +} + +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +{ + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs * attrs2 = dynamic_cast(j->second.e); + if (!attrs2) dupAttr(attrPath, pos, j->second.pos); + attrs = attrs2; + } else + dupAttr(attrPath, pos, j->second.pos); + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + ExprAttrs *nested = new ExprAttrs; + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(e); + auto jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs.emplace(ad.first, ad.second); + } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); + e->setName(i->symbol); + } + } else { + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); + } +} + +inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) +{ + std::sort(formals->formals.begin(), formals->formals.end(), + [] (const auto & a, const auto & b) { + return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); + }); + + std::optional> duplicate; + for (size_t i = 0; i + 1 < formals->formals.size(); i++) { + if (formals->formals[i].name != formals->formals[i + 1].name) + continue; + std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; + duplicate = std::min(thisDup, duplicate.value_or(thisDup)); + } + if (duplicate) + throw ParseError({ + .msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), + .pos = positions[duplicate->second] + }); + + if (arg && formals->has(arg)) + throw ParseError({ + .msg = HintFmt("duplicate formal function argument '%1%'", symbols[arg]), + .pos = positions[pos] + }); + + return formals; +} + +inline Expr * ParserState::stripIndentation(const PosIdx pos, + std::vector>> && es) +{ + if (es.empty()) return new ExprString(""); + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto & [i_pos, i] : es) { + auto * str = std::get_if(&i); + if (!str || !str->hasIndentation) { + /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + continue; + } + for (size_t j = 0; j < str->l; ++j) { + if (atStartOfLine) { + if (str->p[j] == ' ') + curIndent++; + else if (str->p[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + } else if (str->p[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + auto * es2 = new std::vector>; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + auto i = es.begin(); + const auto trimExpr = [&] (Expr * e) { + atStartOfLine = false; + curDropped = 0; + es2->emplace_back(i->first, e); + }; + const auto trimString = [&] (const StringToken & t) { + std::string s2; + for (size_t j = 0; j < t.l; ++j) { + if (atStartOfLine) { + if (t.p[j] == ' ') { + if (curDropped++ >= minIndent) + s2 += t.p[j]; + } + else if (t.p[j] == '\n') { + curDropped = 0; + s2 += t.p[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += t.p[j]; + } + } else { + s2 += t.p[j]; + if (t.p[j] == '\n') atStartOfLine = true; + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) + s2 = std::string(s2, 0, p + 1); + } + + es2->emplace_back(i->first, new ExprString(std::move(s2))); + }; + for (; i != es.end(); ++i, --n) { + std::visit(overloaded { trimExpr, trimString }, i->second); + } + + /* If this is a single string, then don't do a concatenation. */ + if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { + auto *const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); +} + +inline PosIdx ParserState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.first_line, loc.first_column); +} + +} diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5c7fc265dc7..6255442037d 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -5,9 +5,9 @@ %defines /* %no-lines */ %parse-param { void * scanner } -%parse-param { nix::ParseData * data } +%parse-param { nix::ParserState * state } %lex-param { void * scanner } -%lex-param { nix::ParseData * data } +%lex-param { nix::ParserState * state } %expect 1 %expect-rr 1 @@ -18,6 +18,7 @@ #include +#include "finally.hh" #include "util.hh" #include "users.hh" @@ -25,38 +26,26 @@ #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" +#include "parser-state.hh" -namespace nix { +#define YYLTYPE ::nix::ParserLocation +#define YY_DECL int yylex \ + (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) - struct ParseData - { - EvalState & state; - SymbolTable & symbols; - Expr * result; - SourcePath basePath; - PosTable::Origin origin; - std::optional error; - }; +namespace nix { - struct ParserFormals { - std::vector formals; - bool ellipsis = false; - }; +Expr * parseExprFromBuf( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + SymbolTable & symbols, + PosTable & positions, + const ref rootFS, + const Expr::AstSymbols & astSymbols); } -// using C a struct allows us to avoid having to define the special -// members that using string_view here would implicitly delete. -struct StringToken { - const char * p; - size_t l; - bool hasIndentation; - operator std::string_view() const { return {p, l}; } -}; - -#define YY_DECL int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) - #endif } @@ -70,242 +59,17 @@ YY_DECL; using namespace nix; - -namespace nix { +#define CUR_POS state->at(*yylocp) -static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(state.symbols, attrPath), state.positions[prevPos]), - .errPos = state.positions[pos] + .msg = HintFmt(error), + .pos = state->positions[state->at(*loc)] }); } -static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos) -{ - throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), - .errPos = state.positions[pos] - }); -} - - -static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, - Expr * e, const PosIdx pos, const nix::EvalState & state) -{ - AttrPath::iterator i; - // All attrpaths have at least one attr - assert(!attrPath.empty()); - // Checking attrPath validity. - // =========================== - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - if (!j->second.inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(state, attrPath, pos, j->second.pos); - attrs = attrs2; - } else - dupAttr(state, attrPath, pos, j->second.pos); - } else { - ExprAttrs * nested = new ExprAttrs; - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; - } - } else { - ExprAttrs *nested = new ExprAttrs; - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; - } - } - // Expr insertion. - // ========================== - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(state, ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - } - jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); - } else { - dupAttr(state, attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } - } else { - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); - } -} - - -static Formals * toFormals(ParseData & data, ParserFormals * formals, - PosIdx pos = noPos, Symbol arg = {}) -{ - std::sort(formals->formals.begin(), formals->formals.end(), - [] (const auto & a, const auto & b) { - return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); - }); - - std::optional> duplicate; - for (size_t i = 0; i + 1 < formals->formals.size(); i++) { - if (formals->formals[i].name != formals->formals[i + 1].name) - continue; - std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; - duplicate = std::min(thisDup, duplicate.value_or(thisDup)); - } - if (duplicate) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]), - .errPos = data.state.positions[duplicate->second] - }); - - Formals result; - result.ellipsis = formals->ellipsis; - result.formals = std::move(formals->formals); - - if (arg && result.has(arg)) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), - .errPos = data.state.positions[pos] - }); - - delete formals; - return new Formals(std::move(result)); -} - - -static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, - std::vector>> && es) -{ - if (es.empty()) return new ExprString(""); - - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ - size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->l; ++j) { - if (atStartOfLine) { - if (str->p[j] == ' ') - curIndent++; - else if (str->p[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->p[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } - } - } - - /* Strip spaces from each line. */ - auto * es2 = new std::vector>; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (Expr * e) { - atStartOfLine = false; - curDropped = 0; - es2->emplace_back(i->first, e); - }; - const auto trimString = [&] (const StringToken & t) { - std::string s2; - for (size_t j = 0; j < t.l; ++j) { - if (atStartOfLine) { - if (t.p[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.p[j]; - } - else if (t.p[j] == '\n') { - curDropped = 0; - s2 += t.p[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.p[j]; - } - } else { - s2 += t.p[j]; - if (t.p[j] == '\n') atStartOfLine = true; - } - } - - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); - } - - es2->emplace_back(i->first, new ExprString(std::move(s2))); - }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, i->second); - } - - /* If this is a single string, then don't do a concatenation. */ - if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { - auto *const result = (*es2)[0].second; - delete es2; - return result; - } - return new ExprConcatStrings(pos, true, es2); -} - - -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylocp, data) - - -} - - -void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) -{ - data->error = { - .msg = hintfmt(error), - .errPos = data->state.positions[makeCurPos(*loc, data)] - }; -} - %} @@ -314,17 +78,17 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err nix::Expr * e; nix::ExprList * list; nix::ExprAttrs * attrs; - nix::ParserFormals * formals; + nix::Formals * formals; nix::Formal * formal; nix::NixInt n; nix::NixFloat nf; - StringToken id; // !!! -> Symbol - StringToken path; - StringToken uri; - StringToken str; + nix::StringToken id; // !!! -> Symbol + nix::StringToken path; + nix::StringToken uri; + nix::StringToken str; std::vector * attrNames; std::vector> * string_parts; - std::vector>> * ind_string_parts; + std::vector>> * ind_string_parts; struct { nix::Expr * e; bool appendSlash; @@ -345,11 +109,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %type attr %token ID %token STR IND_STR -%token INT -%token FLOAT +%token INT_LIT +%token FLOAT_LIT %token PATH HPATH SPATH PATH_END %token URI -%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS @@ -369,34 +133,34 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %% -start: expr { data->result = $1; }; +start: expr { state->result = $1; }; expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } + { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, toFormals(*data, $2), $5); } + { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); } | '{' formals '}' '@' ID ':' expr_function { - auto arg = data->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7); + auto arg = state->symbols.create($5); + $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); } | ID '@' '{' formals '}' ':' expr_function { - auto arg = data->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7); + auto arg = state->symbols.create($1); + $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } | WITH expr ';' expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } - | LET binds IN expr_function + | LET binds IN_KW expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = data->state.positions[CUR_POS] + .msg = HintFmt("dynamic attributes not allowed in let"), + .pos = state->positions[CUR_POS] }); $$ = new ExprLet($2, $4); } @@ -410,24 +174,24 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(makeCurPos(@2, data), $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); } + | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); } | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(makeCurPos(@2, data), $1, $3); } + { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector >({{state->at(@1), $1}, {state->at(@3), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); } | expr_app ; @@ -450,7 +214,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); } | expr_simple ; @@ -460,35 +224,35 @@ expr_simple if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0) $$ = new ExprPos(CUR_POS); else - $$ = new ExprVar(CUR_POS, data->symbols.create($1)); + $$ = new ExprVar(CUR_POS, state->symbols.create($1)); } - | INT { $$ = new ExprInt($1); } - | FLOAT { $$ = new ExprFloat($1); } + | INT_LIT { $$ = new ExprInt($1); } + | FLOAT_LIT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2)); + $$ = state->stripIndentation(CUR_POS, std::move(*$2)); delete $2; } | path_start PATH_END { $$ = $1.e; } | path_start string_parts_interpolated PATH_END { if ($1.appendSlash) $2->insert($2->begin(), {noPos, new ExprString("/")}); - $2->insert($2->begin(), {makeCurPos(@1, data), $1.e}); + $2->insert($2->begin(), {state->at(@1), $1.e}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { std::string path($1.p + 1, $1.l - 2); $$ = new ExprCall(CUR_POS, - new ExprVar(data->symbols.create("__findFile")), - {new ExprVar(data->symbols.create("__nixPath")), + new ExprVar(state->s.findFile), + {new ExprVar(state->s.nixPath), new ExprString(std::move(path))}); } | URI { static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ - .msg = hintfmt("URL literals are disabled"), - .errPos = data->state.positions[CUR_POS] + .msg = HintFmt("URL literals are disabled"), + .pos = state->positions[CUR_POS] }); $$ = new ExprString(std::string($1)); } @@ -496,7 +260,7 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.create("body")); } + { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' { $3->recursive = true; $$ = $3; } | '{' binds '}' @@ -512,13 +276,13 @@ string_parts string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } - | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(makeCurPos(@1, data), $2); } + { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(state->at(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$ = new std::vector>; - $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1))); - $$->emplace_back(makeCurPos(@2, data), $3); + $$->emplace_back(state->at(@1), new ExprString(std::string($1))); + $$->emplace_back(state->at(@2), $3); } ; @@ -531,8 +295,8 @@ path_start root filesystem accessor, rather than the accessor of the current Nix expression. */ hasPrefix(path, "/") - ? SourcePath{data->state.rootFS, CanonPath(path)} - : SourcePath{data->basePath.accessor, CanonPath(path, data->basePath.path)}), + ? SourcePath{state->rootFS, CanonPath(path)} + : SourcePath{state->basePath.accessor, CanonPath(path, state->basePath.path)}), .appendSlash = hasSuffix(path, "/") }; } @@ -544,24 +308,24 @@ path_start ); } CanonPath path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = {.e = new ExprPath(data->state.rootPath(std::move(path))), .appendSlash = true}; + $$ = {.e = new ExprPath(SourcePath{state->rootFS, std::move(path)}), .appendSlash = true}; } ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } | { $$ = new std::vector>>; } ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; } + : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(data->state, i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); - auto pos = makeCurPos(@3, data); + state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos); + auto pos = state->at(@3); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } delete $3; @@ -571,48 +335,48 @@ binds /* !!! Should ensure sharing of the expression in $4. */ for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); + state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6))); } delete $6; } - | { $$ = new ExprAttrs(makeCurPos(@0, data)); } + | { $$ = new ExprAttrs(state->at(@0)); } ; attrs - : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); } + : attrs attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($2))); } | attrs string_attr { $$ = $1; ExprString * str = dynamic_cast($2); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = data->state.positions[makeCurPos(@2, data)] + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = state->positions[state->at(@2)] }); } | { $$ = new AttrPath; } ; attrpath - : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); } + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); } | attrpath '.' string_attr { $$ = $1; ExprString * str = dynamic_cast($3); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($3)); } - | attr { $$ = new std::vector; $$->push_back(AttrName(data->symbols.create($1))); } + | attr { $$ = new std::vector; $$->push_back(AttrName(state->symbols.create($1))); } | string_attr { $$ = new std::vector; ExprString *str = dynamic_cast($1); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($1)); @@ -638,226 +402,52 @@ formals : formal ',' formals { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } | formal - { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } + { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } | - { $$ = new ParserFormals; $$->ellipsis = false; } + { $$ = new Formals; $$->ellipsis = false; } | ELLIPSIS - { $$ = new ParserFormals; $$->ellipsis = true; } + { $$ = new Formals; $$->ellipsis = true; } ; formal - : ID { $$ = new Formal{CUR_POS, data->symbols.create($1), 0}; } - | ID '?' expr { $$ = new Formal{CUR_POS, data->symbols.create($1), $3}; } + : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; } + | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; } ; %% - -#include -#include -#include -#include - #include "eval.hh" -#include "filetransfer.hh" -#include "fetchers.hh" -#include "fs-input-accessor.hh" -#include "tarball.hh" -#include "store-api.hh" -#include "flake/flake.hh" -#include "fs-input-accessor.hh" -#include "memory-input-accessor.hh" namespace nix { -unsigned long Expr::nrExprs = 0; - -Expr * EvalState::parse( +Expr * parseExprFromBuf( char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, - std::shared_ptr & staticEnv) + SymbolTable & symbols, + PosTable & positions, + const ref rootFS, + const Expr::AstSymbols & astSymbols) { yyscan_t scanner; - ParseData data { - .state = *this, + ParserState state { .symbols = symbols, + .positions = positions, .basePath = basePath, .origin = {origin}, + .rootFS = rootFS, + .s = astSymbols, }; yylex_init(&scanner); - yy_scan_buffer(text, length, scanner); - int res = yyparse(scanner, &data); - yylex_destroy(scanner); - - if (res) throw ParseError(data.error.value()); - - data.result->bindVars(*this, staticEnv); - - return data.result; -} - - -SourcePath resolveExprPath(SourcePath path) -{ - unsigned int followCount = 0, maxFollow = 1024; - - /* If `path' is a symlink, follow it. This is so that relative - path references work. */ - while (!path.path.isRoot()) { - // Basic cycle/depth limit to avoid infinite loops. - if (++followCount >= maxFollow) - throw Error("too many symbolic links encountered while traversing the path '%s'", path); - auto p = path.parent().resolveSymlinks() + path.baseName(); - if (p.lstat().type != InputAccessor::tSymlink) break; - path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; - } - - /* If `path' refers to a directory, append `/default.nix'. */ - if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) - return path + "default.nix"; - - return path; -} - - -Expr * EvalState::parseExprFromFile(const SourcePath & path) -{ - return parseExprFromFile(path, staticBaseEnv); -} - - -Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) -{ - auto buffer = path.resolveSymlinks().readFile(); - // readFile hopefully have left some extra space for terminators - buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) -{ - auto s = make_ref(std::move(s_)); - s->append("\0\0", 2); - return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); -} - + Finally _destroy([&] { yylex_destroy(scanner); }); -Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) -{ - return parseExprFromString(std::move(s), basePath, staticBaseEnv); -} - - -Expr * EvalState::parseStdin() -{ - //Activity act(*logger, lvlTalkative, "parsing standard input"); - auto buffer = drainFD(0); - // drainFD should have left some extra space for terminators - buffer.append("\0\0", 2); - auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); -} - - -SourcePath EvalState::findFile(const std::string_view path) -{ - return findFile(searchPath, path); -} - - -SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) -{ - for (auto & i : searchPath.elements) { - auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); - - if (!suffixOpt) continue; - auto suffix = *suffixOpt; - - auto rOpt = resolveSearchPathPath(i.path); - if (!rOpt) continue; - auto r = *rOpt; - - auto res = r + CanonPath(suffix); - if (res.pathExists()) return res; - } - - if (hasPrefix(path, "nix/")) - return {corepkgsFS, CanonPath(path.substr(3))}; - - debugThrow(ThrownError({ - .msg = hintfmt(evalSettings.pureEval - ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" - : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", - path), - .errPos = positions[pos] - }), 0, 0); -} - - -std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) -{ - auto & value = value0.s; - auto i = searchPathResolved.find(value); - if (i != searchPathResolved.end()) return i->second; - - std::optional res; - - if (EvalSettings::isPseudoUrl(value)) { - try { - auto accessor = fetchers::downloadTarball( - EvalSettings::resolvePseudoUrl(value)).accessor; - registerAccessor(accessor); - res.emplace(SourcePath(accessor)); - } catch (Error & e) { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) - }); - } - } - - else if (hasPrefix(value, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", value); - auto [accessor, _] = flakeRef.resolve(store).lazyFetch(store); - res.emplace(SourcePath(accessor)); - } - - else { - auto path = rootPath(CanonPath::fromCwd(value)); - - /* Allow access to paths in the search path. */ - if (initAccessControl) { - allowPath(path.path.abs()); - if (store->isInStore(path.path.abs())) { - try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); - for (auto & p : closure) - allowPath(p); - } catch (InvalidPath &) { } - } - } - - if (path.pathExists()) - res.emplace(path); - else { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) - }); - } - } - - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); + yy_scan_buffer(text, length, scanner); + yyparse(scanner, &state); - searchPathResolved.emplace(value, res); - return res; + return state.result; } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index f6cfb8eee8d..512be710d60 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,6 +1,5 @@ #include "eval.hh" #include "util.hh" -#include "fs-input-accessor.hh" namespace nix { @@ -9,6 +8,11 @@ SourcePath EvalState::rootPath(CanonPath path) return {rootFS, std::move(path)}; } +SourcePath EvalState::rootPath(PathView path) +{ + return {rootFS, CanonPath(absPath(path))}; +} + void EvalState::registerAccessor(ref accessor) { inputAccessors.emplace(accessor->number, accessor); @@ -29,11 +33,11 @@ std::string EvalState::encodePath(const SourcePath & path) SourcePath EvalState::decodePath(std::string_view s, PosIdx pos) { if (!hasPrefix(s, "/")) - error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); + error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); if (hasPrefix(s, virtualPathMarker)) { auto fail = [s, pos, this]() { - error("cannot decode virtual path '%s'", s).atPos(pos).debugThrow(); + error("cannot decode virtual path '%s'", s).atPos(pos).debugThrow(); }; s = s.substr(virtualPathMarker.size()); diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh new file mode 100644 index 00000000000..9949f1dc51d --- /dev/null +++ b/src/libexpr/pos-idx.hh @@ -0,0 +1,48 @@ +#pragma once + +#include + +namespace nix { + +class PosIdx +{ + friend class PosTable; + +private: + uint32_t id; + + explicit PosIdx(uint32_t id) + : id(id) + { + } + +public: + PosIdx() + : id(0) + { + } + + explicit operator bool() const + { + return id > 0; + } + + bool operator<(const PosIdx other) const + { + return id < other.id; + } + + bool operator==(const PosIdx other) const + { + return id == other.id; + } + + bool operator!=(const PosIdx other) const + { + return id != other.id; + } +}; + +inline PosIdx noPos = {}; + +} diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh new file mode 100644 index 00000000000..1decf3c85a7 --- /dev/null +++ b/src/libexpr/pos-table.hh @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +#include "chunked-vector.hh" +#include "pos-idx.hh" +#include "position.hh" + +namespace nix { + +class PosTable +{ +public: + class Origin + { + friend PosTable; + private: + // must always be invalid by default, add() replaces this with the actual value. + // subsequent add() calls use this index as a token to quickly check whether the + // current origins.back() can be reused or not. + mutable uint32_t idx = std::numeric_limits::max(); + + // Used for searching in PosTable::[]. + explicit Origin(uint32_t idx) + : idx(idx) + , origin{std::monostate()} + { + } + + public: + const Pos::Origin origin; + + Origin(Pos::Origin origin) + : origin(origin) + { + } + }; + + struct Offset + { + uint32_t line, column; + }; + +private: + std::vector origins; + ChunkedVector offsets; + +public: + PosTable() + : offsets(1024) + { + origins.reserve(1024); + } + + PosIdx add(const Origin & origin, uint32_t line, uint32_t column) + { + const auto idx = offsets.add({line, column}).second; + if (origins.empty() || origins.back().idx != origin.idx) { + origin.idx = idx; + origins.push_back(origin); + } + return PosIdx(idx + 1); + } + + Pos operator[](PosIdx p) const + { + if (p.id == 0 || p.id > offsets.size()) + return {}; + const auto idx = p.id - 1; + /* we want the last key <= idx, so we'll take prev(first key > idx). + this is guaranteed to never rewind origin.begin because the first + key is always 0. */ + const auto pastOrigin = std::upper_bound( + origins.begin(), origins.end(), Origin(idx), [](const auto & a, const auto & b) { return a.idx < b.idx; }); + const auto origin = *std::prev(pastOrigin); + const auto offset = offsets[idx]; + return {offset.line, offset.column, origin.origin}; + } +}; + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ce93cf824fd..00ec6c87756 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -16,6 +16,7 @@ #include "value-to-xml.hh" #include "primops.hh" #include "fs-input-accessor.hh" +#include "fetch-to-store.hh" #include #include @@ -38,10 +39,6 @@ namespace nix { * Miscellaneous *************************************************************/ - -InvalidPathError::InvalidPathError(const Path & path) : - EvalError("path '%s' is not valid", path), path(path) {} - StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; @@ -50,7 +47,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { if (!store->isValidPath(p)) - debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); + error(store->printStorePath(p)).debugThrow(); }; std::visit(overloaded { [&](const NixStringContextElem::Built & b) { @@ -77,21 +74,22 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (drvs.empty()) return {}; if (!evalSettings.enableImportFromDerivation) - debugThrowLastTrace(Error( + error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - drvs.begin()->to_string(*store))); + drvs.begin()->to_string(*store) + ).debugThrow(); /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->buildPaths(buildReqs); + buildStore->buildPaths(buildReqs, bmNormal, store); + + StorePathSet outputsToCopyAndAllow; for (auto & drv : drvs) { - auto outputs = resolveDerivedPath(*store, drv); + auto outputs = resolveDerivedPath(*buildStore, drv, &*store); for (auto & [outputName, outputPath] : outputs) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(store->toRealPath(outputPath)); + outputsToCopyAndAllow.insert(outputPath); /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -101,12 +99,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = drv.drvPath, .output = outputName, }).render(), - store->printStorePath(outputPath) + buildStore->printStorePath(outputPath) ); } } } + if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); + for (auto & outputPath : outputsToCopyAndAllow) { + /* Add the output of this derivations to the allowed + paths. */ + allowPath(outputPath); + } + return res; } @@ -217,7 +222,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - auto staticEnv = std::make_shared(false, state.staticBaseEnv.get(), vScope->attrs->size()); + auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { @@ -338,16 +343,16 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror())); + state.error("could not open '%1%': %2%", path, dlerror()).debugThrow(); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - state.debugThrowLastTrace(EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message)); + state.error("could not load symbol '%1%' from '%2%': %3%", sym, path, message).debugThrow(); else - state.debugThrowLastTrace(EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path)); + state.error("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path).debugThrow(); } (func)(state, v); @@ -364,7 +369,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) - state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); + state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); NixStringContext context; auto program = state.coerceToString(pos, *elems[0], context, "while evaluating the first element of the argument passed to builtins.exec", @@ -379,7 +384,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) try { auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { - state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow(); + state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow(); } auto output = runProgram(program, true, commandArgs); @@ -445,9 +450,7 @@ static RegisterPrimOp primop_isNull({ .doc = R"( Return `true` if *e* evaluates to `null`, and `false` otherwise. - > **Warning** - > - > This function is *deprecated*; just write `e == null` instead. + This is equivalent to `e == null`. )", .fun = prim_isNull, }); @@ -583,7 +586,7 @@ struct CompareValues if (v1->type() == nInt && v2->type() == nFloat) return v1->integer < v2->fpoint; if (v1->type() != v2->type()) - state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" @@ -593,7 +596,7 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return v1->string_view().compare(v2->string_view()) < 0; + return strcmp(v1->c_str(), v2->c_str()) < 0; case nPath: // Note: we don't take the accessor into account // since it's not obvious how to compare them in a @@ -611,7 +614,7 @@ struct CompareValues } } default: - state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); #pragma GCC diagnostic pop } } catch (Error & e) { @@ -638,7 +641,7 @@ static Bindings::iterator getAttr( { Bindings::iterator value = attrSet->find(attrSym); if (value == attrSet->end()) { - state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); + state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); } return value; } @@ -758,8 +761,8 @@ static RegisterPrimOp primop_break({ if (state.debugRepl && !state.debugTraces.empty()) { auto error = Error(ErrorInfo { .level = lvlInfo, - .msg = hintfmt("breakpoint reached"), - .errPos = state.positions[pos], + .msg = HintFmt("breakpoint reached"), + .pos = state.positions[pos], }); auto & dt = state.debugTraces.front(); @@ -769,8 +772,8 @@ static RegisterPrimOp primop_break({ // If the user elects to quit the repl, throw an exception. throw Error(ErrorInfo{ .level = lvlInfo, - .msg = hintfmt("quit the debugger"), - .errPos = nullptr, + .msg = HintFmt("quit the debugger"), + .pos = nullptr, }); } } @@ -789,9 +792,9 @@ static RegisterPrimOp primop_abort({ .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; - auto s = state.decodePaths(*state.coerceToString(pos, *args[0], context, - "while evaluating the error message passed to 'builtins.abort'")); - state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to 'builtins.abort'").toOwned(); + state.error("evaluation aborted with the following error message: '%1%'", s).debugThrow(); } }); @@ -808,9 +811,9 @@ static RegisterPrimOp primop_throw({ .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; - auto s = state.decodePaths(*state.coerceToString(pos, *args[0], context, - "while evaluating the error message passed to 'builtin.throw'")); - state.debugThrowLastTrace(ThrownError(s)); + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to 'builtin.throw'").toOwned(); + state.error(s).debugThrow(); } }); @@ -821,10 +824,10 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * v = *args[1]; } catch (Error & e) { NixStringContext context; - auto message = state.decodePaths(*state.coerceToString(pos, *args[0], context, + auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to 'builtins.addErrorContext'", - false, false)); - e.addTrace(nullptr, hintfmt(message), true); + false, false).toOwned(); + e.addTrace(nullptr, HintFmt(message), true); throw; } } @@ -998,7 +1001,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu if (args[0]->type() == nString) printError("trace: %1%", state.decodePaths(args[0]->string_view())); else - printError("trace: %1%", printValue(state, *args[0])); + printError("trace: %1%", ValuePrinter(state, *args[0])); state.forceValue(*args[1], pos); v = *args[1]; } @@ -1075,7 +1078,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * * often results from the composition of several functions * (derivationStrict, derivation, mkDerivation, mkPythonModule, etc.) */ - e.addTrace(nullptr, hintfmt( + e.addTrace(nullptr, HintFmt( "while evaluating derivation '%s'\n" " whose name attribute is located at %s", drvName, pos), true); @@ -1089,9 +1092,10 @@ drvName, Bindings * attrs, Value & v) /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; std::optional jsonObject; + auto pos = v.determinePos(noPos); auto attr = attrs->find(state.sStructuredAttrs); if (attr != attrs->end() && - state.forceBool(*attr->value, noPos, + state.forceBool(*attr->value, pos, "while evaluating the `__structuredAttrs` " "attribute passed to builtins.derivationStrict")) jsonObject = json::object(); @@ -1100,7 +1104,7 @@ drvName, Bindings * attrs, Value & v) bool ignoreNulls = false; attr = attrs->find(state.sIgnoreNulls); if (attr != attrs->end()) - ignoreNulls = state.forceBool(*attr->value, noPos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); + ignoreNulls = state.forceBool(*attr->value, pos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ Derivation drv; @@ -1129,37 +1133,33 @@ drvName, Bindings * attrs, Value & v) experimentalFeatureSettings.require(Xp::DynamicDerivations); ingestionMethod = TextIngestionMethod {}; } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), - .errPos = state.positions[noPos] - })); + state.error( + "invalid value '%s' for 'outputHashMode' attribute", s + ).atPos(v).debugThrow(); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("duplicate derivation output '%1%'", j), - .errPos = state.positions[noPos] - })); + state.error("duplicate derivation output '%1%'", j) + .atPos(v) + .debugThrow(); /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid derivation output name 'drv'" ), - .errPos = state.positions[noPos] - })); + state.error("invalid derivation output name 'drv'") + .atPos(v) + .debugThrow(); outputs.insert(j); } if (outputs.empty()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation cannot have an empty set of outputs"), - .errPos = state.positions[noPos] - })); + state.error("derivation cannot have an empty set of outputs") + .atPos(v) + .debugThrow(); }; try { @@ -1168,16 +1168,16 @@ drvName, Bindings * attrs, Value & v) const std::string_view context_below(""); if (ignoreNulls) { - state.forceValue(*i->value, noPos); + state.forceValue(*i->value, pos); if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) { + if (i->name == state.sContentAddressed && state.forceBool(*i->value, pos, context_below)) { contentAddressed = true; experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) { + else if (i->name == state.sImpure && state.forceBool(*i->value, pos, context_below)) { isImpure = true; experimentalFeatureSettings.require(Xp::ImpureDerivations); } @@ -1185,9 +1185,9 @@ drvName, Bindings * attrs, Value & v) /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { - state.forceList(*i->value, noPos, context_below); + state.forceList(*i->value, pos, context_below); for (auto elem : i->value->listItems()) { - auto s = state.coerceToString(noPos, *elem, context, + auto s = state.coerceToString(pos, *elem, context, "while evaluating an element of the argument list", true).toOwned(); drv.args.push_back(s); @@ -1202,29 +1202,29 @@ drvName, Bindings * attrs, Value & v) if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, noPos, context); + (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, noPos, context_below); + drv.builder = state.forceString(*i->value, context, pos, context_below); else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, noPos, context_below); + drv.platform = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, noPos, context_below); + outputHash = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, noPos, context_below); + outputHashAlgo = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, noPos, context_below)); + handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below)); else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, noPos, context_below); + state.forceList(*i->value, pos, context_below); Strings ss; for (auto elem : i->value->listItems()) - ss.emplace_back(state.forceStringNoCtx(*elem, noPos, context_below)); + ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below)); handleOutputs(ss); } } else { - auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned(); + auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = std::move(s); else if (i->name == state.sSystem) drv.platform = std::move(s); @@ -1239,7 +1239,7 @@ drvName, Bindings * attrs, Value & v) } catch (Error & e) { e.addTrace(state.positions[i->pos], - hintfmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName), + HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName), true); throw; } @@ -1282,16 +1282,14 @@ drvName, Bindings * attrs, Value & v) /* Do we have all required attributes? */ if (drv.builder == "") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("required attribute 'builder' missing"), - .errPos = state.positions[noPos] - })); + state.error("required attribute 'builder' missing") + .atPos(v) + .debugThrow(); if (drv.platform == "") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("required attribute 'system' missing"), - .errPos = state.positions[noPos] - })); + state.error("required attribute 'system' missing") + .atPos(v) + .debugThrow(); /* Check whether the derivation name is valid. */ if (isDerivation(drvName) && @@ -1299,10 +1297,10 @@ drvName, Bindings * attrs, Value & v) outputs.size() == 1 && *(outputs.begin()) == "out")) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), - .errPos = state.positions[noPos] - })); + state.error( + "derivation names are allowed to end in '%s' only if they produce a single derivation file", + drvExtension + ).atPos(v).debugThrow(); } if (outputHash) { @@ -1311,10 +1309,9 @@ drvName, Bindings * attrs, Value & v) Ignore `__contentAddressed` because fixed output derivations are already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - state.debugThrowLastTrace(Error({ - .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), - .errPos = state.positions[noPos] - })); + state.error( + "multiple outputs are not supported in fixed-output derivations" + ).atPos(v).debugThrow(); auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo)); @@ -1333,10 +1330,8 @@ drvName, Bindings * attrs, Value & v) else if (contentAddressed || isImpure) { if (contentAddressed && isImpure) - throw EvalError({ - .msg = hintfmt("derivation cannot be both content-addressed and impure"), - .errPos = state.positions[noPos] - }); + state.error("derivation cannot be both content-addressed and impure") + .atPos(v).debugThrow(); auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); @@ -1377,10 +1372,10 @@ drvName, Bindings * attrs, Value & v) for (auto & i : outputs) { auto h = get(hashModulo.hashes, i); if (!h) - throw AssertionError({ - .msg = hintfmt("derivation produced no hash for output '%s'", i), - .errPos = state.positions[noPos], - }); + state.error( + "derivation produced no hash for output '%s'", + i + ).atPos(v).debugThrow(); auto outPath = state.store->makeOutputPath(i, *h, drvName); drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign( @@ -1487,10 +1482,10 @@ static RegisterPrimOp primop_toPath({ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { if (evalSettings.pureEval) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"), - .errPos = state.positions[pos] - })); + state.error( + "'%s' is not allowed in pure evaluation mode", + "builtins.storePath" + ).atPos(pos).debugThrow(); NixStringContext context; auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path; @@ -1500,10 +1495,8 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, if (!state.store->isStorePath(path.abs())) path = CanonPath(canonPath(path.abs(), true)); if (!state.store->isInStore(path.abs())) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = state.positions[pos] - })); + state.error("path '%1%' is not in the Nix store", path) + .atPos(pos).debugThrow(); auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); @@ -1618,7 +1611,10 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[0]); auto s = path.readFile(); if (s.find((char) 0) != std::string::npos) - state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); + state.error( + "the contents of the file '%1%' cannot be represented as a Nix string", + path + ).atPos(pos).debugThrow(); StorePathSet refs; if (state.store->isInStore(path.path.abs())) { try { @@ -1676,10 +1672,11 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V auto rewrites = state.realiseContext(context); path = rewriteStrings(path, rewrites); } catch (InvalidPathError & e) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), - .errPos = state.positions[pos] - })); + state.error( + "cannot find '%1%', since path '%2%' is not valid", + path, + e.path + ).atPos(pos).debugThrow(); } searchPath.elements.emplace_back(SearchPath::Elem { @@ -1748,10 +1745,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); std::optional ha = parseHashAlgo(algo); if (!ha) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash algo '%1%'", algo), - .errPos = state.positions[pos] - })); + state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); auto path = realisePath(state, pos, *args[1]); @@ -1819,7 +1813,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va // detailed node info quickly in this case we produce a thunk to // query the file type lazily. auto epath = state.allocValue(); - epath->mkPath(path + name); + epath->mkPath(path / name); if (!readFileType) readFileType = &state.getBuiltin("readFileType"); attr.mkApp(readFileType, epath); @@ -1881,7 +1875,7 @@ static RegisterPrimOp primop_outputOf({ For instance, ```nix builtins.outputOf - (builtins.outputOf myDrv "out) + (builtins.outputOf myDrv "out") "out" ``` will return a placeholder for the output of the output of `myDrv`. @@ -2071,18 +2065,23 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val if (auto p = std::get_if(&c.raw)) refs.insert(p->path); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt( - "in 'toFile': the file named '%1%' must not contain a reference " - "to a derivation but contains (%2%)", - name, c.to_string()), - .errPos = state.positions[pos] - })); + state.error( + "files created by %1% may not reference derivations, but %2% references %3%", + "builtins.toFile", + name, + c.to_string() + ).atPos(pos).debugThrow(); } auto storePath = settings.readOnlyMode - ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair); + ? state.store->makeFixedOutputPathFromCA(name, TextInfo { + .hash = hashString(HashAlgorithm::SHA256, contents), + .references = std::move(refs), + }) + : ({ + StringSource s { contents }; + state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + }); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths @@ -2242,9 +2241,12 @@ static void addPath( // store on-demand. if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair); + auto dstPath = fetchToStore(*state.store, path.resolveSymlinks(), name, method, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) - state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); + state.error( + "store path mismatch in (possibly filtered) path added from '%s'", + path + ).atPos(pos).debugThrow(); state.allowAndSetStorePathString(dstPath, v); } else state.allowAndSetStorePathString(*expectedStorePath, v); @@ -2344,16 +2346,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - })); + state.error( + "unsupported argument '%1%' to 'addPath'", + state.symbols[attr.name] + ).atPos(attr.pos).debugThrow(); } if (!path) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"), - .errPos = state.positions[pos] - })); + state.error( + "missing required 'path' attribute in the first argument to builtins.path" + ).atPos(pos).debugThrow(); if (name.empty()) name = path->baseName(); @@ -2414,7 +2415,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); + [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; }); } static RegisterPrimOp primop_attrNames({ @@ -2771,10 +2772,7 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg return; } if (!args[0]->isLambda()) - state.debugThrowLastTrace(TypeError({ - .msg = hintfmt("'functionArgs' requires a function"), - .errPos = state.positions[pos] - })); + state.error("'functionArgs' requires a function").atPos(pos).debugThrow(); if (!args[0]->lambda.fun->hasFormals()) { v.mkAttrs(&state.emptyBindings); @@ -2944,10 +2942,10 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val { state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); if (n < 0 || (unsigned int) n >= list.listSize()) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("list index %1% is out of bounds", n), - .errPos = state.positions[pos] - })); + state.error( + "list index %1% is out of bounds", + n + ).atPos(pos).debugThrow(); state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -2992,10 +2990,7 @@ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value { state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); if (args[0]->listSize() == 0) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("'tail' called on an empty list"), - .errPos = state.positions[pos] - })); + state.error("'tail' called on an empty list").atPos(pos).debugThrow(); state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) @@ -3252,7 +3247,7 @@ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Va auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); if (len < 0) - state.error("cannot create list of size %1%", len).debugThrow(); + state.error("cannot create list of size %1%", len).atPos(pos).debugThrow(); // More strict than striclty (!) necessary, but acceptable // as evaluating map without accessing any values makes little sense. @@ -3569,10 +3564,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixFloat f2 = state.forceFloat(*args[1], pos, "while evaluating the second operand of the division"); if (f2 == 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("division by zero"), - .errPos = state.positions[pos] - })); + state.error("division by zero").atPos(pos).debugThrow(); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first operand of the division") / f2); @@ -3581,10 +3573,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("overflow in integer division"), - .errPos = state.positions[pos] - })); + state.error("overflow in integer division").atPos(pos).debugThrow(); v.mkInt(i1 / i2); } @@ -3713,16 +3702,26 @@ static RegisterPrimOp primop_toString({ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + + if (start < 0) + state.error("negative start position in 'substring'").atPos(pos).debugThrow(); + + int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); + + // Special-case on empty substring to avoid O(n) strlen + // This allows for the use of empty substrings to efficently capture string context + if (len == 0) { + state.forceValue(*args[2], pos); + if (args[2]->type() == nString) { + v.mkString("", args[2]->context()); + return; + } + } + NixStringContext context; auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); - if (start < 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("negative start position in 'substring'"), - .errPos = state.positions[pos] - })); - v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context); } @@ -3770,10 +3769,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); std::optional ha = parseHashAlgo(algo); if (!ha) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash algo '%1%'", algo), - .errPos = state.positions[pos] - })); + state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); @@ -3939,15 +3935,13 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("memory limit exceeded by regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("invalid regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } } @@ -4043,15 +4037,13 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("memory limit exceeded by regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("invalid regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } } @@ -4127,7 +4119,9 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.replaceStrings"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.replaceStrings"); if (args[0]->listSize() != args[1]->listSize()) - state.error("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths").atPos(pos).debugThrow(); + state.error( + "'from' and 'to' arguments passed to builtins.replaceStrings have different lengths" + ).atPos(pos).debugThrow(); std::vector from; from.reserve(args[0]->listSize()); @@ -4396,13 +4390,16 @@ void EvalState::createBaseEnv() .impureOnly = true, }); - if (!evalSettings.pureEval) { - v.mkString(settings.thisSystem.get()); - } + if (!evalSettings.pureEval) + v.mkString(evalSettings.getCurrentSystem()); addConstant("__currentSystem", v, { .type = nString, .doc = R"( - The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system). + The value of the + [`eval-system`](@docroot@/command-ref/conf-file.md#conf-eval-system) + or else + [`system`](@docroot@/command-ref/conf-file.md#conf-system) + configuration option. It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index db940f277b2..1eec8b316f9 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -98,30 +98,30 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V auto contextSize = context.size(); if (contextSize != 1) { - throw EvalError({ - .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), - .errPos = state.positions[pos] - }); + state.error( + "context of string '%s' must have exactly one element, but has %d", + *s, + contextSize + ).atPos(pos).debugThrow(); } NixStringContext context2 { (NixStringContextElem { std::visit(overloaded { [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { if (!c.path.isDerivation()) { - throw EvalError({ - .msg = hintfmt("path '%s' is not a derivation", - state.store->printStorePath(c.path)), - .errPos = state.positions[pos], - }); + state.error( + "path '%s' is not a derivation", + state.store->printStorePath(c.path) + ).atPos(pos).debugThrow(); } return NixStringContextElem::DrvDeep { .drvPath = c.path, }; }, [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { - throw EvalError({ - .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), - .errPos = state.positions[pos], - }); + state.error( + "`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", + c.output + ).atPos(pos).debugThrow(); }, [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { /* Reuse original item because we want this to be idempotent. */ @@ -261,10 +261,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto & i : *args[1]->attrs) { const auto & name = state.symbols[i.name]; if (!state.store->isStorePath(name)) - throw EvalError({ - .msg = hintfmt("context key '%s' is not a store path", name), - .errPos = state.positions[i.pos] - }); + state.error( + "context key '%s' is not a store path", + name + ).atPos(i.pos).debugThrow(); auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) state.store->ensurePath(namePath); @@ -281,10 +281,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) { if (!isDerivation(name)) { - throw EvalError({ - .msg = hintfmt("tried to add all-outputs context of %s, which is not a derivation, to a string", name), - .errPos = state.positions[i.pos] - }); + state.error( + "tried to add all-outputs context of %s, which is not a derivation, to a string", + name + ).atPos(i.pos).debugThrow(); } context.emplace(NixStringContextElem::DrvDeep { .drvPath = namePath, @@ -296,10 +296,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar if (iter != i.value->attrs->end()) { state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context"); if (iter->value->listSize() && !isDerivation(name)) { - throw EvalError({ - .msg = hintfmt("tried to add derivation output context of %s, which is not a derivation, to a string", name), - .errPos = state.positions[i.pos] - }); + state.error( + "tried to add derivation output context of %s, which is not a derivation, to a string", + name + ).atPos(i.pos).debugThrow(); } for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 27147a5d13c..f51a6465dc7 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -23,20 +23,20 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); if (toPathMaybe && *toPathMaybe != rewrittenPath) throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + .msg = HintFmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", state.store->printStorePath(fromPath), state.store->printStorePath(rewrittenPath), state.store->printStorePath(*toPathMaybe)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); if (!toPathMaybe) throw Error({ - .msg = hintfmt( + .msg = HintFmt( "rewriting '%s' to content-addressed form yielded '%s'\n" "Use this value for the 'toPath' attribute passed to 'fetchClosure'", state.store->printStorePath(fromPath), state.store->printStorePath(rewrittenPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -50,11 +50,11 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor // We don't perform the rewriting when outPath already exists, as an optimisation. // However, we can quickly detect a mistake if the toPath is input addressed. throw Error({ - .msg = hintfmt( + .msg = HintFmt( "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", state.store->printStorePath(toPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -73,14 +73,14 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos if (!info->isContentAddressed(*state.store)) { throw Error({ - .msg = hintfmt( + .msg = HintFmt( "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" "If you do intend to fetch an input-addressed store path, add\n\n" " inputAddressed = true;\n\n" "to the 'fetchClosure' arguments.\n\n" "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", state.store->printStorePath(fromPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -99,11 +99,11 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId if (info->isContentAddressed(*state.store)) { throw Error({ - .msg = hintfmt( + .msg = HintFmt( "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", state.store->printStorePath(fromPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -153,15 +153,15 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else throw Error({ - .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), + .pos = state.positions[pos] }); } if (!fromPath) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), + .pos = state.positions[pos] }); bool inputAddressed = inputAddressedMaybe.value_or(false); @@ -169,17 +169,17 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg if (inputAddressed) { if (toPath) throw Error({ - .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", + .msg = HintFmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", "inputAddressed", "toPath"), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } if (!fromStoreUrl) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), + .pos = state.positions[pos] }); auto parsedURL = parseURL(*fromStoreUrl); @@ -188,14 +188,14 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg parsedURL.scheme != "https" && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) throw Error({ - .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), - .errPos = state.positions[pos] + .msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), + .pos = state.positions[pos] }); if (!parsedURL.query.empty()) throw Error({ - .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), - .errPos = state.positions[pos] + .msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), + .pos = state.positions[pos] }); auto fromStore = openStore(parsedURL.to_string()); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index abbc95aaa65..3864dae7da6 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -38,17 +38,11 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.fetchMercurial"); else - throw EvalError({ - .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - }); + state.error("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]).atPos(attr.pos).debugThrow(); } if (url.empty()) - throw EvalError({ - .msg = hintfmt("'url' argument required"), - .errPos = state.positions[pos] - }); + state.error("'url' argument required").atPos(pos).debugThrow(); } else url = state.coerceToString(pos, *args[0], context, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cc7dc407015..ed60b2e38d7 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,3 +1,4 @@ +#include "libfetchers/attrs.hh" #include "primops.hh" #include "eval-inline.hh" #include "eval-settings.hh" @@ -8,6 +9,7 @@ #include "tarball.hh" #include "url.hh" #include "value-to-json.hh" +#include "fetch-to-store.hh" #include #include @@ -23,7 +25,7 @@ static void emitTreeAttrs( bool emptyRevFallback, bool forceDirty) { - auto attrs = state.buildBindings(10); + auto attrs = state.buildBindings(100); setOutPath(attrs.alloc(state.sOutPath)); @@ -113,16 +115,14 @@ static void fetchTree( if (auto aType = args[0]->attrs->get(state.sType)) { if (type) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unexpected attribute 'type'"), - .errPos = state.positions[pos] - })); + state.error( + "unexpected attribute 'type'" + ).atPos(pos).debugThrow(); type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree"); } else if (!type) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'type' is missing in call to 'fetchTree'" + ).atPos(pos).debugThrow(); attrs.emplace("type", type.value()); @@ -147,16 +147,19 @@ static void fetchTree( attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); } else - state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", - state.symbols[attr.name], showType(*attr.value))); + state.error("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); + } + + if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { + attrs.emplace("exportIgnore", Explicit{true}); } if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'name' isn’t supported in call to 'fetchTree'" + ).atPos(pos).debugThrow(); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { @@ -168,13 +171,15 @@ static void fetchTree( fetchers::Attrs attrs; attrs.emplace("type", "git"); attrs.emplace("url", fixGitURL(url)); + if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { + attrs.emplace("exportIgnore", Explicit{true}); + } input = fetchers::Input::fromAttrs(std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), - .errPos = state.positions[pos] - })); + state.error( + "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" + ).atPos(pos).debugThrow(); input = fetchers::Input::fromURL(url); } } @@ -182,8 +187,16 @@ static void fetchTree( if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isLocked()) - state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); + if (evalSettings.pureEval && !input.isLocked()) { + auto fetcher = "fetchTree"; + if (params.isFetchGit) + fetcher = "fetchGit"; + + state.error( + "in pure evaluation mode, %s requires a locked input", + fetcher + ).atPos(pos).debugThrow(); + } state.checkURI(input.toURLString()); @@ -456,17 +469,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%s' to '%s'", n, who), - .errPos = state.positions[attr.pos] - })); + state.error("unsupported argument '%s' to '%s'", n, who) + .atPos(pos).debugThrow(); } if (!url) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'url' argument required"), - .errPos = state.positions[pos] - })); + state.error( + "'url' argument required").atPos(pos).debugThrow(); } else url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); @@ -479,7 +488,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v name = baseNameOf(*url); if (evalSettings.pureEval && !expectedHash) - state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); + state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) { @@ -501,16 +510,22 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(*url).accessor->fetchToStore(state.store, CanonPath::root, name) + ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, name) : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); - if (hash != *expectedHash) - state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true))); + if (hash != *expectedHash) { + state.error( + "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", + *url, + expectedHash->to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true) + ).withExitStatus(102) + .debugThrow(); + } } state.allowAndSetStorePathString(storePath, v); @@ -630,10 +645,16 @@ static RegisterPrimOp primop_fetchGit({ A Boolean parameter that specifies whether submodules should be checked out. + - `exportIgnore` (default: `true`) + + A Boolean parameter that specifies whether `export-ignore` from `.gitattributes` should be applied. + This approximates part of the `git archive` behavior. + + Enabling this option is not recommended because it is unknown whether the Git developers commit to the reproducibility of `export-ignore` in newer Git versions. + - `shallow` (default: `false`) - A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed. - This still performs a full clone of what is available on the remote. + Make a shallow clone when fetching the Git tree. - `allRefs` diff --git a/src/libexpr/primops/filterPath.cc b/src/libexpr/primops/filterPath.cc index 1a7e5ba1f54..5c77e1c0a4d 100644 --- a/src/libexpr/primops/filterPath.cc +++ b/src/libexpr/primops/filterPath.cc @@ -22,7 +22,7 @@ struct FilterPathInputAccessor : CachingFilteringInputAccessor if (!path.isRoot() && !isAllowed(*path.parent())) return false; // Note that unlike 'builtins.{path,filterSource}', we don't // pass the prefix to the filter function. - return state.callPathFilter(filterFun, {next, prefix + path}, path.abs(), pos); + return state.callPathFilter(filterFun, {next, prefix / path}, path.abs(), pos); } }; @@ -46,29 +46,21 @@ static void prim_filterPath(EvalState & state, PosIdx pos, Value * * args, Value filterFun = attr.value; } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%1%' to 'filterPath'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - })); + state.error( + "unsupported argument '%1%' to 'filterPath'", state.symbols[attr.name]) + .atPos(attr.pos).debugThrow(); } if (!path) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'path' required"), - .errPos = state.positions[pos] - })); + state.error("'path' required").atPos(pos).debugThrow(); if (!filterFun) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'filter' required"), - .errPos = state.positions[pos] - })); + state.error("'filter' required").atPos(pos).debugThrow(); if (!context.empty()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'path' argument to 'filterPath' cannot have a context"), - .errPos = state.positions[pos] - })); + state.error( + "'path' argument to 'filterPath' cannot have a context") + .atPos(pos).debugThrow(); auto accessor = make_ref(state, pos, *path, filterFun); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 2f4d4022e8f..94be7960a06 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -83,10 +83,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V try { visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); } catch (std::exception & e) { // TODO: toml::syntax_error - throw EvalError({ - .msg = hintfmt("while parsing a TOML string: %s", e.what()), - .errPos = state.positions[pos] - }); + state.error("while parsing TOML: %s", e.what()).atPos(pos).debugThrow(); } } diff --git a/src/libexpr/primops/patch.cc b/src/libexpr/primops/patch.cc index 547157afd83..27ccfeda497 100644 --- a/src/libexpr/primops/patch.cc +++ b/src/libexpr/primops/patch.cc @@ -1,4 +1,5 @@ #include "primops.hh" +#include "patching-input-accessor.hh" namespace nix { @@ -16,10 +17,9 @@ static void prim_patch(EvalState & state, const PosIdx pos, Value * * args, Valu auto check = [&]() { if (!patches.empty()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'builtins.patch' does not support both 'patches' and 'patchFiles'"), - .errPos = state.positions[attr.pos] - })); + state.error( + "'builtins.patch' does not support both 'patches' and 'patchFiles'") + .atPos(attr.pos).debugThrow(); }; if (n == "src") { @@ -50,17 +50,15 @@ static void prim_patch(EvalState & state, const PosIdx pos, Value * * args, Valu } else - throw Error({ - .msg = hintfmt("attribute '%s' isn't supported in call to 'builtins.patch'", n), - .errPos = state.positions[pos] - }); + state.error( + "attribute '%s' isn't supported in call to 'builtins.patch'", n) + .atPos(pos).debugThrow(); } if (!src) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'src' is missing in call to 'builtins.patch'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'src' is missing in call to 'builtins.patch'") + .atPos(pos).debugThrow(); if (!src->path.isRoot()) throw UnimplementedError("applying patches to a non-root path ('%s') is not yet supported", src->path); diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc new file mode 100644 index 00000000000..521250cecdb --- /dev/null +++ b/src/libexpr/print-ambiguous.cc @@ -0,0 +1,101 @@ +#include "print-ambiguous.hh" +#include "print.hh" +#include "signals.hh" +#include "eval.hh" + +namespace nix { + +// See: https://github.com/NixOS/nix/issues/9730 +void printAmbiguous( + Value &v, + const SymbolTable &symbols, + std::ostream &str, + std::set *seen, + int depth) +{ + checkInterrupt(); + + if (depth <= 0) { + str << "«too deep»"; + return; + } + switch (v.type()) { + case nInt: + str << v.integer; + break; + case nBool: + printLiteralBool(str, v.boolean); + break; + case nString: + printLiteralString(str, v.string_view()); + break; + case nPath: + str << v.path().to_string(); // !!! escaping? + break; + case nNull: + str << "null"; + break; + case nAttrs: { + if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second) + str << "«repeated»"; + else { + str << "{ "; + for (auto & i : v.attrs->lexicographicOrder(symbols)) { + str << symbols[i->name] << " = "; + printAmbiguous(*i->value, symbols, str, seen, depth - 1); + str << "; "; + } + str << "}"; + } + break; + } + case nList: + if (seen && v.listSize() && !seen->insert(v.listElems()).second) + str << "«repeated»"; + else { + str << "[ "; + for (auto v2 : v.listItems()) { + if (v2) + printAmbiguous(*v2, symbols, str, seen, depth - 1); + else + str << "(nullptr)"; + str << " "; + } + str << "]"; + } + break; + case nThunk: + if (!v.isBlackhole()) { + str << ""; + } else { + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + } + break; + case nFunction: + if (v.isLambda()) { + str << ""; + } else if (v.isPrimOp()) { + str << ""; + } else if (v.isPrimOpApp()) { + str << ""; + } + break; + case nExternal: + str << *v.external; + break; + case nFloat: + str << v.fpoint; + break; + default: + printError("Nix evaluator internal error: printAmbiguous: invalid value type"); + abort(); + } +} + +} diff --git a/src/libexpr/print-ambiguous.hh b/src/libexpr/print-ambiguous.hh new file mode 100644 index 00000000000..50c260a9b84 --- /dev/null +++ b/src/libexpr/print-ambiguous.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "value.hh" + +namespace nix { + +/** + * Print a value in the deprecated format used by `nix-instantiate --eval` and + * `nix-env` (for manifests). + * + * This output can't be changed because it's part of the `nix-instantiate` API, + * but it produces ambiguous output; unevaluated thunks and lambdas (and a few + * other types) are printed as Nix path syntax like ``. + * + * See: https://github.com/NixOS/nix/issues/9730 + */ +void printAmbiguous( + Value &v, + const SymbolTable &symbols, + std::ostream &str, + std::set *seen, + int depth); + +} diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh new file mode 100644 index 00000000000..6c5e80c61d8 --- /dev/null +++ b/src/libexpr/print-options.hh @@ -0,0 +1,92 @@ +#pragma once +/** + * @file + * @brief Options for printing Nix values. + */ + +#include + +namespace nix { + +/** + * Options for printing Nix values. + */ +struct PrintOptions +{ + /** + * If true, output ANSI color sequences. + */ + bool ansiColors = false; + + /** + * If true, force values. + */ + bool force = false; + + /** + * If true and `force` is set, print derivations as + * `«derivation /nix/store/...»` instead of as attribute sets. + */ + bool derivationPaths = false; + + /** + * If true, track which values have been printed and skip them on + * subsequent encounters. Useful for self-referential values. + */ + bool trackRepeated = true; + + /** + * Maximum depth to evaluate to. + */ + size_t maxDepth = std::numeric_limits::max(); + + /** + * Maximum number of attributes in attribute sets to print. + * + * Note that this is a limit for the entire print invocation, not for each + * attribute set encountered. + */ + size_t maxAttrs = std::numeric_limits::max(); + + /** + * Maximum number of list items to print. + * + * Note that this is a limit for the entire print invocation, not for each + * list encountered. + */ + size_t maxListItems = std::numeric_limits::max(); + + /** + * Maximum string length to print. + */ + size_t maxStringLength = std::numeric_limits::max(); + + /** + * Indentation width for pretty-printing. + * + * If set to 0 (the default), values are not pretty-printed. + */ + size_t prettyIndent = 0; + + /** + * True if pretty-printing is enabled. + */ + inline bool shouldPrettyPrint() + { + return prettyIndent > 0; + } +}; + +/** + * `PrintOptions` for unknown and therefore potentially large values in error messages, + * to avoid printing "too much" output. + */ +static PrintOptions errorPrintOptions = PrintOptions { + .ansiColors = true, + .maxDepth = 10, + .maxAttrs = 10, + .maxListItems = 10, + .maxStringLength = 1024 +}; + +} diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 53ba70bdd49..9d280f6239c 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -1,24 +1,67 @@ -#include "print.hh" +#include #include +#include "print.hh" +#include "ansicolor.hh" +#include "signals.hh" +#include "store-api.hh" +#include "terminal.hh" +#include "english.hh" +#include "eval.hh" + namespace nix { +void printElided( + std::ostream & output, + unsigned int value, + const std::string_view single, + const std::string_view plural, + bool ansiColors) +{ + if (ansiColors) + output << ANSI_FAINT; + output << "«"; + pluralize(output, value, single, plural); + output << " elided»"; + if (ansiColors) + output << ANSI_NORMAL; +} + + std::ostream & -printLiteralString(std::ostream & str, const std::string_view string) +printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors) { + size_t charsPrinted = 0; + if (ansiColors) + str << ANSI_MAGENTA; str << "\""; for (auto i = string.begin(); i != string.end(); ++i) { + if (charsPrinted >= maxLength) { + str << "\" "; + printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors); + return str; + } + if (*i == '\"' || *i == '\\') str << "\\" << *i; else if (*i == '\n') str << "\\n"; else if (*i == '\r') str << "\\r"; else if (*i == '\t') str << "\\t"; else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; else str << *i; + charsPrinted++; } str << "\""; + if (ansiColors) + str << ANSI_NORMAL; return str; } +std::ostream & +printLiteralString(std::ostream & str, const std::string_view string) +{ + return printLiteralString(str, string, std::numeric_limits::max(), false); +} + std::ostream & printLiteralBool(std::ostream & str, bool boolean) { @@ -90,5 +133,481 @@ printAttributeName(std::ostream & str, std::string_view name) { return str; } +bool isImportantAttrName(const std::string& attrName) +{ + return attrName == "type" || attrName == "_type"; +} + +typedef std::pair AttrPair; + +struct ImportantFirstAttrNameCmp +{ + + bool operator()(const AttrPair& lhs, const AttrPair& rhs) const + { + auto lhsIsImportant = isImportantAttrName(lhs.first); + auto rhsIsImportant = isImportantAttrName(rhs.first); + return std::forward_as_tuple(!lhsIsImportant, lhs.first) + < std::forward_as_tuple(!rhsIsImportant, rhs.first); + } +}; + +typedef std::set ValuesSeen; +typedef std::vector> AttrVec; + +class Printer +{ +private: + std::ostream & output; + EvalState & state; + PrintOptions options; + std::optional seen; + size_t attrsPrinted = 0; + size_t listItemsPrinted = 0; + std::string indent; + + void increaseIndent() + { + if (options.shouldPrettyPrint()) { + indent.append(options.prettyIndent, ' '); + } + } + + void decreaseIndent() + { + if (options.shouldPrettyPrint()) { + assert(indent.size() >= options.prettyIndent); + indent.resize(indent.size() - options.prettyIndent); + } + } + + /** + * Print a space (for separating items or attributes). + * + * If pretty-printing is enabled, a newline and the current `indent` is + * printed instead. + */ + void printSpace(bool prettyPrint) + { + if (prettyPrint) { + output << "\n" << indent; + } else { + output << " "; + } + } + + void printRepeated() + { + if (options.ansiColors) + output << ANSI_MAGENTA; + output << "«repeated»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printNullptr() + { + if (options.ansiColors) + output << ANSI_MAGENTA; + output << "«nullptr»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printElided(unsigned int value, const std::string_view single, const std::string_view plural) + { + ::nix::printElided(output, value, single, plural, options.ansiColors); + } + + void printInt(Value & v) + { + if (options.ansiColors) + output << ANSI_CYAN; + output << v.integer; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printFloat(Value & v) + { + if (options.ansiColors) + output << ANSI_CYAN; + output << v.fpoint; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printBool(Value & v) + { + if (options.ansiColors) + output << ANSI_CYAN; + printLiteralBool(output, v.boolean); + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printString(Value & v) + { + printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors); + } + + void printPath(Value & v) + { + if (options.ansiColors) + output << ANSI_GREEN; + output << v.path().to_string(); // !!! escaping? + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printNull() + { + if (options.ansiColors) + output << ANSI_CYAN; + output << "null"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printDerivation(Value & v) + { + try { + Bindings::iterator i = v.attrs->find(state.sDrvPath); + NixStringContext context; + std::string storePath; + if (i != v.attrs->end()) + storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); + + if (options.ansiColors) + output << ANSI_GREEN; + output << "«derivation"; + if (!storePath.empty()) { + output << " " << storePath; + } + output << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } catch (Error & e) { + printError_(e); + } + } + + bool shouldPrettyPrintAttrs(AttrVec & v) + { + if (!options.shouldPrettyPrint() || v.empty()) { + return false; + } + + // Pretty-print attrsets with more than one item. + if (v.size() > 1) { + return true; + } + + auto item = v[0].second; + if (!item) { + return true; + } + + // Pretty-print single-item attrsets only if they contain nested + // structures. + auto itemType = item->type(); + return itemType == nList || itemType == nAttrs || itemType == nThunk; + } + + void printAttrs(Value & v, size_t depth) + { + if (seen && !seen->insert(v.attrs).second) { + printRepeated(); + return; + } + + if (options.force && options.derivationPaths && state.isDerivation(v)) { + printDerivation(v); + } else if (depth < options.maxDepth) { + increaseIndent(); + output << "{"; + + AttrVec sorted; + for (auto & i : *v.attrs) + sorted.emplace_back(std::pair(state.symbols[i.name], i.value)); + + if (options.maxAttrs == std::numeric_limits::max()) + std::sort(sorted.begin(), sorted.end()); + else + std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp()); + + auto prettyPrint = shouldPrettyPrintAttrs(sorted); + + for (auto & i : sorted) { + printSpace(prettyPrint); + + if (attrsPrinted >= options.maxAttrs) { + printElided(sorted.size() - attrsPrinted, "attribute", "attributes"); + break; + } + + printAttributeName(output, i.first); + output << " = "; + print(*i.second, depth + 1); + output << ";"; + attrsPrinted++; + } + + decreaseIndent(); + printSpace(prettyPrint); + output << "}"; + } else { + output << "{ ... }"; + } + } + + bool shouldPrettyPrintList(std::span list) + { + if (!options.shouldPrettyPrint() || list.empty()) { + return false; + } + + // Pretty-print lists with more than one item. + if (list.size() > 1) { + return true; + } + + auto item = list[0]; + if (!item) { + return true; + } + + // Pretty-print single-item lists only if they contain nested + // structures. + auto itemType = item->type(); + return itemType == nList || itemType == nAttrs || itemType == nThunk; + } + + void printList(Value & v, size_t depth) + { + if (seen && v.listSize() && !seen->insert(&v).second) { + printRepeated(); + return; + } + + if (depth < options.maxDepth) { + increaseIndent(); + output << "["; + auto listItems = v.listItems(); + auto prettyPrint = shouldPrettyPrintList(listItems); + for (auto elem : listItems) { + printSpace(prettyPrint); + + if (listItemsPrinted >= options.maxListItems) { + printElided(listItems.size() - listItemsPrinted, "item", "items"); + break; + } + + if (elem) { + print(*elem, depth + 1); + } else { + printNullptr(); + } + listItemsPrinted++; + } + + decreaseIndent(); + printSpace(prettyPrint); + output << "]"; + } else { + output << "[ ... ]"; + } + } + + void printFunction(Value & v) + { + if (options.ansiColors) + output << ANSI_BLUE; + output << "«"; + + if (v.isLambda()) { + output << "lambda"; + if (v.lambda.fun) { + if (v.lambda.fun->name) { + output << " " << state.symbols[v.lambda.fun->name]; + } + + std::ostringstream s; + s << state.positions[v.lambda.fun->pos]; + output << " @ " << filterANSIEscapes(s.str()); + } + } else if (v.isPrimOp()) { + if (v.primOp) + output << *v.primOp; + else + output << "primop"; + } else if (v.isPrimOpApp()) { + output << "partially applied "; + auto primOp = v.primOpAppPrimOp(); + if (primOp) + output << *primOp; + else + output << "primop"; + } else { + abort(); + } + + output << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printThunk(Value & v) + { + if (v.isBlackhole()) { + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + if (options.ansiColors) + output << ANSI_RED; + output << "«potential infinite recursion»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } else if (v.isThunk() || v.isApp()) { + if (options.ansiColors) + output << ANSI_MAGENTA; + output << "«thunk»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } else { + abort(); + } + } + + void printExternal(Value & v) + { + v.external->print(output); + } + + void printUnknown() + { + if (options.ansiColors) + output << ANSI_RED; + output << "«unknown»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printError_(Error & e) + { + if (options.ansiColors) + output << ANSI_RED; + output << "«error: " << filterANSIEscapes(e.info().msg.str(), true) << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void print(Value & v, size_t depth) + { + output.flush(); + checkInterrupt(); + + if (options.force) { + try { + state.forceValue(v, v.determinePos(noPos)); + } catch (Error & e) { + printError_(e); + return; + } + } + + switch (v.type()) { + + case nInt: + printInt(v); + break; + + case nFloat: + printFloat(v); + break; + + case nBool: + printBool(v); + break; + + case nString: + printString(v); + break; + + case nPath: + printPath(v); + break; + + case nNull: + printNull(); + break; + + case nAttrs: + printAttrs(v, depth); + break; + + case nList: + printList(v, depth); + break; + + case nFunction: + printFunction(v); + break; + + case nThunk: + printThunk(v); + break; + + case nExternal: + printExternal(v); + break; + + default: + printUnknown(); + break; + } + } + +public: + Printer(std::ostream & output, EvalState & state, PrintOptions options) + : output(output), state(state), options(options) { } + + void print(Value & v) + { + attrsPrinted = 0; + listItemsPrinted = 0; + indent.clear(); + + if (options.trackRepeated) { + seen.emplace(); + } else { + seen.reset(); + } + + ValuesSeen seen; + print(v, 0); + } +}; + +void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options) +{ + Printer(output, state, options).print(v); +} + +std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) +{ + printValue(printer.state, output, printer.value, printer.options); + return output; +} + +template<> +HintFmt & HintFmt::operator%(const ValuePrinter & value) +{ + fmt % value; + return *this; +} } diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh index 3b72ae2016a..7ddda81b88f 100644 --- a/src/libexpr/print.hh +++ b/src/libexpr/print.hh @@ -9,46 +9,83 @@ #include +#include "fmt.hh" +#include "print-options.hh" + namespace nix { - /** - * Print a string as a Nix string literal. - * - * Quotes and fairly minimal escaping are added. - * - * @param s The logical string - */ - std::ostream & printLiteralString(std::ostream & o, std::string_view s); - inline std::ostream & printLiteralString(std::ostream & o, const char * s) { - return printLiteralString(o, std::string_view(s)); - } - inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { - return printLiteralString(o, std::string_view(s)); - } - - /** Print `true` or `false`. */ - std::ostream & printLiteralBool(std::ostream & o, bool b); - - /** - * Print a string as an attribute name in the Nix expression language syntax. - * - * Prints a quoted string if necessary. - */ - std::ostream & printAttributeName(std::ostream & o, std::string_view s); - - /** - * Returns `true' is a string is a reserved keyword which requires quotation - * when printing attribute set field names. - */ - bool isReservedKeyword(const std::string_view str); - - /** - * Print a string as an identifier in the Nix expression language syntax. - * - * FIXME: "identifier" is ambiguous. Identifiers do not have a single - * textual representation. They can be used in variable references, - * let bindings, left-hand sides or attribute names in a select - * expression, or something else entirely, like JSON. Use one of the - * `print*` functions instead. - */ - std::ostream & printIdentifier(std::ostream & o, std::string_view s); + +class EvalState; +struct Value; + +/** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param o The output stream to print to + * @param s The logical string + */ +std::ostream & printLiteralString(std::ostream & o, std::string_view s); +inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); +} +inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); +} + +/** Print `true` or `false`. */ +std::ostream & printLiteralBool(std::ostream & o, bool b); + +/** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ +std::ostream & printAttributeName(std::ostream & o, std::string_view s); + +/** + * Returns `true' is a string is a reserved keyword which requires quotation + * when printing attribute set field names. + */ +bool isReservedKeyword(const std::string_view str); + +/** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ +std::ostream & printIdentifier(std::ostream & o, std::string_view s); + +void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions {}); + +/** + * A partially-applied form of `printValue` which can be formatted using `<<` + * without allocating an intermediate string. + */ +class ValuePrinter { + friend std::ostream & operator << (std::ostream & output, const ValuePrinter & printer); +private: + EvalState & state; + Value & value; + PrintOptions options; + +public: + ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions {}) + : state(state), value(value), options(options) { } +}; + +std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer); + + +/** + * `ValuePrinter` does its own ANSI formatting, so we don't color it + * magenta. + */ +template<> +HintFmt & HintFmt::operator%(const ValuePrinter & value); + } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 74b3ebf136b..3f877a7fd4f 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -64,7 +64,7 @@ json printValueAsJSON(EvalState & state, bool strict, out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); } catch (Error & e) { e.addTrace(state.positions[a.pos], - hintfmt("while evaluating attribute '%1%'", j)); + HintFmt("while evaluating attribute '%1%'", j)); throw; } } @@ -80,8 +80,8 @@ json printValueAsJSON(EvalState & state, bool strict, try { out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); } catch (Error & e) { - e.addTrace({}, - hintfmt("while evaluating list element at index %1%", i)); + e.addTrace(state.positions[pos], + HintFmt("while evaluating list element at index %1%", i)); throw; } i++; @@ -99,13 +99,12 @@ json printValueAsJSON(EvalState & state, bool strict, case nThunk: case nFunction: - auto e = TypeError({ - .msg = hintfmt("cannot convert %1% to JSON", showType(v)), - .errPos = state.positions[v.determinePos(pos)] - }); - e.addTrace(state.positions[pos], hintfmt("message for the trace")); - state.debugThrowLastTrace(e); - throw e; + state.error( + "cannot convert %1% to JSON", + showType(v) + ) + .atPos(v.determinePos(pos)) + .debugThrow(); } return out; } @@ -119,7 +118,8 @@ void printValueAsJSON(EvalState & state, bool strict, json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore) const { - state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); + state.error("cannot convert %1% to JSON", showType()) + .debugThrow(); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 30b3d49343e..e7aea49492a 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -8,6 +8,8 @@ #include "symbol-table.hh" #include "value/context.hh" #include "input-accessor.hh" +#include "source-path.hh" +#include "print-options.hh" #if HAVE_BOEHMGC #include @@ -32,7 +34,6 @@ typedef enum { tThunk, tApp, tLambda, - tBlackhole, tPrimOp, tPrimOpApp, tExternal, @@ -62,6 +63,7 @@ class Bindings; struct Env; struct Expr; struct ExprLambda; +struct ExprBlackHole; struct PrimOp; class Symbol; class PosIdx; @@ -69,7 +71,7 @@ struct Pos; class StorePath; class EvalState; class XMLWriter; - +class Printer; typedef int64_t NixInt; typedef double NixFloat; @@ -81,6 +83,7 @@ typedef double NixFloat; class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); + friend class Printer; protected: /** * Print out the value @@ -102,7 +105,7 @@ class ExternalValueBase * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; /** * Compare to another value of the same type. Defaults to uncomparable, @@ -138,11 +141,9 @@ private: friend std::string showType(const Value & v); - void print(const SymbolTable &symbols, std::ostream &str, std::set *seen, int depth) const; - public: - void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const; + void print(EvalState &state, std::ostream &str, PrintOptions options = PrintOptions {}); // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's @@ -151,7 +152,7 @@ public: // type() == nThunk inline bool isThunk() const { return internalType == tThunk; }; inline bool isApp() const { return internalType == tApp; }; - inline bool isBlackhole() const { return internalType == tBlackhole; }; + inline bool isBlackhole() const; // type() == nFunction inline bool isLambda() const { return internalType == tLambda; }; @@ -248,7 +249,7 @@ public: case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; case tExternal: return nExternal; case tFloat: return nFloat; - case tThunk: case tApp: case tBlackhole: return nThunk; + case tThunk: case tApp: return nThunk; } if (invalidIsThunk) return nThunk; @@ -356,21 +357,22 @@ public: lambda.fun = f; } - inline void mkBlackhole() - { - internalType = tBlackhole; - // Value will be overridden anyways - } + inline void mkBlackhole(); void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { internalType = tPrimOpApp; - app.left = l; - app.right = r; + primOpApp.left = l; + primOpApp.right = r; } + /** + * For a `tPrimOpApp` value, get the original `PrimOp` value. + */ + PrimOp * primOpAppPrimOp() const; + inline void mkExternal(ExternalValueBase * e) { clearValue(); @@ -447,6 +449,20 @@ public: }; +extern ExprBlackHole eBlackHole; + +bool Value::isBlackhole() const +{ + return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole; +} + +void Value::mkBlackhole() +{ + internalType = tThunk; + thunk.expr = (Expr*) &eBlackHole; +} + + #if HAVE_BOEHMGC typedef std::vector> ValueVector; typedef std::map, traceable_allocator>> ValueMap; diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 51fd30a44cb..7f23cd3a43f 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -19,8 +19,8 @@ public: : Error("") { raw = raw_; - auto hf = hintfmt(args...); - err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw); + auto hf = HintFmt(args...); + err.msg = HintFmt("Bad String Context element: %1%: %2%", Uncolored(hf.str()), raw); } }; diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 63b05bdab19..e071b4717b4 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -106,7 +106,7 @@ struct CacheImpl : Cache } void add( - ref store, + Store & store, const Attrs & inAttrs, const Attrs & infoAttrs, const StorePath & storePath, @@ -115,13 +115,13 @@ struct CacheImpl : Cache _state.lock()->add.use() (attrsToJSON(inAttrs).dump()) (attrsToJSON(infoAttrs).dump()) - (store->printStorePath(storePath)) + (store.printStorePath(storePath)) (locked) (time(0)).exec(); } std::optional> lookup( - ref store, + Store & store, const Attrs & inAttrs) override { if (auto res = lookupExpired(store, inAttrs)) { @@ -134,7 +134,7 @@ struct CacheImpl : Cache } std::optional lookupExpired( - ref store, + Store & store, const Attrs & inAttrs) override { auto state(_state.lock()); @@ -148,19 +148,19 @@ struct CacheImpl : Cache } auto infoJSON = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); + auto storePath = store.parseStorePath(stmt.getStr(1)); auto locked = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); - store->addTempRoot(storePath); - if (!store->isValidPath(storePath)) { + store.addTempRoot(storePath); + if (!store.isValidPath(storePath)) { // FIXME: we could try to substitute 'storePath'. debug("ignoring disappeared cache entry '%s'", inAttrsJSON); return {}; } debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJSON, infoJSON, store->printStorePath(storePath)); + inAttrsJSON, infoJSON, store.printStorePath(storePath)); return Result { .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index f70589267f7..791d77025aa 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -50,14 +50,14 @@ struct Cache /* Old cache for things that have a store path. */ virtual void add( - ref store, + Store & store, const Attrs & inAttrs, const Attrs & infoAttrs, const StorePath & storePath, bool locked) = 0; virtual std::optional> lookup( - ref store, + Store & store, const Attrs & inAttrs) = 0; struct Result @@ -68,7 +68,7 @@ struct Cache }; virtual std::optional lookupExpired( - ref store, + Store & store, const Attrs & inAttrs) = 0; }; diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc new file mode 100644 index 00000000000..c27880662ba --- /dev/null +++ b/src/libfetchers/fetch-to-store.cc @@ -0,0 +1,54 @@ +#include "fetch-to-store.hh" +#include "fetchers.hh" +#include "cache.hh" + +namespace nix { + +StorePath fetchToStore( + Store & store, + const SourcePath & path, + std::string_view name, + ContentAddressMethod method, + PathFilter * filter, + RepairFlag repair) +{ + // FIXME: add an optimisation for the case where the accessor is + // an FSInputAccessor pointing to a store path. + + std::optional cacheKey; + + if (!filter && path.accessor->fingerprint) { + cacheKey = fetchers::Attrs{ + {"_what", "fetchToStore"}, + {"store", store.storeDir}, + {"name", std::string{name}}, + {"fingerprint", *path.accessor->fingerprint}, + {"method", std::string{method.render()}}, + {"path", path.path.abs()} + }; + if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { + debug("store path cache hit for '%s'", path); + return res->second; + } + } else + debug("source path '%s' is uncacheable", path); + + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path)); + + auto filter2 = filter ? *filter : defaultPathFilter; + + auto storePath = + settings.readOnlyMode + ? store.computeStorePath( + name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first + : store.addToStore( + name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair); + + if (cacheKey) + fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); + + return storePath; +} + + +} diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh new file mode 100644 index 00000000000..e5e039340e9 --- /dev/null +++ b/src/libfetchers/fetch-to-store.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "source-path.hh" +#include "store-api.hh" +#include "file-system.hh" +#include "repair-flag.hh" +#include "file-content-address.hh" + +namespace nix { + +/** + * Copy the `path` to the Nix store. + */ +StorePath fetchToStore( + Store & store, + const SourcePath & path, + std::string_view name = "source", + ContentAddressMethod method = FileIngestionMethod::Recursive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5a2936b5b89..36262ba3558 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,6 +1,8 @@ #include "fetchers.hh" #include "store-api.hh" #include "input-accessor.hh" +#include "source-path.hh" +#include "fetch-to-store.hh" #include @@ -170,7 +172,7 @@ std::pair Input::fetchToStore(ref store) const auto [storePath, input] = [&]() -> std::pair { try { auto [accessor, input2] = getAccessor(store); - auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName()); + auto storePath = nix::fetchToStore(*store, SourcePath(accessor), input2.getName()); return {storePath, input2}; } catch (Error & e) { e.addTrace({}, "while fetching the input '%s'", to_string()); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 70a63244dad..93e29d39877 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -183,6 +183,13 @@ struct InputScheme virtual bool isDirect(const Input & input) const { return true; } + /** + * A sufficiently unique string that can be used as a cache key to identify the `input`. + * + * Only known-equivalent inputs should return the same fingerprint. + * + * This is not a stable identifier between Nix versions, but not guaranteed to change either. + */ virtual std::optional getFingerprint(ref store, const Input & input) const { return std::nullopt; } diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc index 581ce3c1d54..087a100aff2 100644 --- a/src/libfetchers/filtering-input-accessor.cc +++ b/src/libfetchers/filtering-input-accessor.cc @@ -5,26 +5,26 @@ namespace nix { std::string FilteringInputAccessor::readFile(const CanonPath & path) { checkAccess(path); - return next->readFile(prefix + path); + return next->readFile(prefix / path); } bool FilteringInputAccessor::pathExists(const CanonPath & path) { - return isAllowed(path) && next->pathExists(prefix + path); + return isAllowed(path) && next->pathExists(prefix / path); } std::optional FilteringInputAccessor::maybeLstat(const CanonPath & path) { checkAccess(path); - return next->maybeLstat(prefix + path); + return next->maybeLstat(prefix / path); } InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path) { checkAccess(path); DirEntries entries; - for (auto & entry : next->readDirectory(prefix + path)) { - if (isAllowed(path + entry.first)) + for (auto & entry : next->readDirectory(prefix / path)) { + if (isAllowed(path / entry.first)) entries.insert(std::move(entry)); } return entries; @@ -33,12 +33,12 @@ InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath std::string FilteringInputAccessor::readLink(const CanonPath & path) { checkAccess(path); - return next->readLink(prefix + path); + return next->readLink(prefix / path); } std::string FilteringInputAccessor::showPath(const CanonPath & path) { - return next->showPath(prefix + path); + return next->showPath(prefix / path); } void FilteringInputAccessor::checkAccess(const CanonPath & path) diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index ebc2115a34e..8a9b206ee51 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -1,11 +1,12 @@ #pragma once #include "input-accessor.hh" +#include "source-path.hh" namespace nix { /** - * A function that should throw an exception of type + * A function that returns an exception of type * `RestrictedPathError` explaining that access to `path` is * forbidden. */ @@ -70,6 +71,9 @@ struct AllowListInputAccessor : public FilteringInputAccessor using FilteringInputAccessor::FilteringInputAccessor; }; +/** + * A wrapping `InputAccessor` mix-in where `isAllowed()` caches the result of virtual `isAllowedUncached()`. + */ struct CachingFilteringInputAccessor : FilteringInputAccessor { std::map cache; diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index c3d8d273c8a..ee24c621a7c 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -6,72 +6,30 @@ namespace nix { struct FSInputAccessor : InputAccessor, PosixSourceAccessor { - CanonPath root; - - FSInputAccessor(const CanonPath & root) - : root(root) - { - displayPrefix = root.isRoot() ? "" : root.abs(); - } - - void readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) override - { - auto absPath = makeAbsPath(path); - PosixSourceAccessor::readFile(absPath, sink, sizeCallback); - } - - bool pathExists(const CanonPath & path) override - { - return PosixSourceAccessor::pathExists(makeAbsPath(path)); - } - - std::optional maybeLstat(const CanonPath & path) override - { - return PosixSourceAccessor::maybeLstat(makeAbsPath(path)); - } - - DirEntries readDirectory(const CanonPath & path) override - { - DirEntries res; - for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path))) - res.emplace(entry); - return res; - } - - std::string readLink(const CanonPath & path) override - { - return PosixSourceAccessor::readLink(makeAbsPath(path)); - } - - CanonPath makeAbsPath(const CanonPath & path) - { - return root + path; - } - - std::optional getPhysicalPath(const CanonPath & path) override - { - return makeAbsPath(path); - } + using PosixSourceAccessor::PosixSourceAccessor; }; -ref makeFSInputAccessor(const CanonPath & root) +ref makeFSInputAccessor() +{ + return make_ref(); +} + +ref makeFSInputAccessor(std::filesystem::path root) { - return make_ref(root); + return make_ref(std::move(root)); } ref makeStorePathAccessor( ref store, const StorePath & storePath) { - return makeFSInputAccessor(CanonPath(store->toRealPath(storePath))); + // FIXME: should use `store->getFSAccessor()` + return makeFSInputAccessor(std::filesystem::path { store->toRealPath(storePath) }); } SourcePath getUnfilteredRootPath(CanonPath path) { - static auto rootFS = makeFSInputAccessor(CanonPath::root); + static auto rootFS = makeFSInputAccessor(); return {rootFS, path}; } diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh index ba5af5887fd..e60906bd827 100644 --- a/src/libfetchers/fs-input-accessor.hh +++ b/src/libfetchers/fs-input-accessor.hh @@ -1,14 +1,16 @@ #pragma once #include "input-accessor.hh" +#include "source-path.hh" namespace nix { class StorePath; class Store; -ref makeFSInputAccessor( - const CanonPath & root); +ref makeFSInputAccessor(); + +ref makeFSInputAccessor(std::filesystem::path root); ref makeStorePathAccessor( ref store, diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 794fcaf31ff..2998575ffc9 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1,13 +1,15 @@ #include "git-utils.hh" +#include "fs-input-accessor.hh" #include "input-accessor.hh" +#include "filtering-input-accessor.hh" #include "cache.hh" #include "finally.hh" #include "processes.hh" #include "signals.hh" #include "users.hh" +#include "fs-sink.hh" -#include - +#include #include #include #include @@ -22,12 +24,11 @@ #include #include -#include "tarfile.hh" -#include - +#include #include #include #include +#include namespace std { @@ -54,6 +55,8 @@ bool operator == (const git_oid & oid1, const git_oid & oid2) namespace nix { +struct GitInputAccessor; + // Some wrapper types that ensure that the git_*_free functions get called. template struct Deleter @@ -137,15 +140,16 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) struct GitRepoImpl : GitRepo, std::enable_shared_from_this { - CanonPath path; + /** Location of the repository on disk. */ + std::filesystem::path path; Repository repo; - GitRepoImpl(CanonPath _path, bool create, bool bare) + GitRepoImpl(std::filesystem::path _path, bool create, bool bare) : path(std::move(_path)) { initLibGit2(); - if (pathExists(path.abs())) { + if (pathExists(path.native())) { if (git_repository_open(Setter(repo), path.c_str())) throw Error("opening Git repository '%s': %s", path, git_error_last()->message); } else { @@ -218,10 +222,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return toHash(*oid); } - std::vector parseSubmodules(const CanonPath & configFile) + std::vector parseSubmodules(const std::filesystem::path & configFile) { GitConfig config; - if (git_config_open_ondisk(Setter(config), configFile.abs().c_str())) + if (git_config_open_ondisk(Setter(config), configFile.c_str())) throw Error("parsing .gitmodules file: %s", git_error_last()->message); ConfigIterator it; @@ -292,8 +296,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this throw Error("getting working directory status: %s", git_error_last()->message); /* Get submodule info. */ - auto modulesFile = path + ".gitmodules"; - if (pathExists(modulesFile.abs())) + auto modulesFile = path / ".gitmodules"; + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; @@ -311,159 +315,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return std::nullopt; } - TarballInfo importTarball(Source & source) override - { - TarArchive archive(source); - - struct PendingDir - { - std::string name; - TreeBuilder builder; - }; - - std::vector pendingDirs; - - auto pushBuilder = [&](std::string name) - { - git_treebuilder * b; - if (git_treebuilder_new(&b, *this, nullptr)) - throw Error("creating a tree builder: %s", git_error_last()->message); - pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) }); - }; - - auto popBuilder = [&]() -> std::pair - { - assert(!pendingDirs.empty()); - auto pending = std::move(pendingDirs.back()); - git_oid oid; - if (git_treebuilder_write(&oid, pending.builder.get())) - throw Error("creating a tree object: %s", git_error_last()->message); - pendingDirs.pop_back(); - return {oid, pending.name}; - }; - - auto addToTree = [&](const std::string & name, const git_oid & oid, git_filemode_t mode) - { - assert(!pendingDirs.empty()); - auto & pending = pendingDirs.back(); - if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode)) - throw Error("adding a file to a tree builder: %s", git_error_last()->message); - }; - - auto updateBuilders = [&](boost::span names) - { - // Find the common prefix of pendingDirs and names. - size_t prefixLen = 0; - for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen) - if (names[prefixLen] != pendingDirs[prefixLen + 1].name) - break; - - // Finish the builders that are not part of the common prefix. - for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) { - auto [oid, name] = popBuilder(); - addToTree(name, oid, GIT_FILEMODE_TREE); - } - - // Create builders for the new directories. - for (auto n = prefixLen; n < names.size(); ++n) - pushBuilder(names[n]); - }; - - pushBuilder(""); - - size_t componentsToStrip = 1; - - time_t lastModified = 0; - - for (;;) { - // FIXME: merge with extract_archive - struct archive_entry * entry; - int r = archive_read_next_header(archive.archive, &entry); - if (r == ARCHIVE_EOF) break; - auto path = archive_entry_pathname(entry); - if (!path) - throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); - if (r == ARCHIVE_WARN) - warn(archive_error_string(archive.archive)); - else - archive.check(r); - - lastModified = std::max(lastModified, archive_entry_mtime(entry)); - - auto pathComponents = tokenizeString>(path, "/"); - - boost::span pathComponents2{pathComponents}; - - if (pathComponents2.size() <= componentsToStrip) continue; - pathComponents2 = pathComponents2.subspan(componentsToStrip); - - updateBuilders( - archive_entry_filetype(entry) == AE_IFDIR - ? pathComponents2 - : pathComponents2.first(pathComponents2.size() - 1)); - - switch (archive_entry_filetype(entry)) { - - case AE_IFDIR: - // Nothing to do right now. - break; - - case AE_IFREG: { - - git_writestream * stream = nullptr; - if (git_blob_create_from_stream(&stream, *this, nullptr)) - throw Error("creating a blob stream object: %s", git_error_last()->message); - - while (true) { - std::vector buf(128 * 1024); - auto n = archive_read_data(archive.archive, buf.data(), buf.size()); - if (n < 0) - throw Error("cannot read file '%s' from tarball", path); - if (n == 0) break; - if (stream->write(stream, (const char *) buf.data(), n)) - throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); - } - - git_oid oid; - if (git_blob_create_from_stream_commit(&oid, stream)) - throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message); - - addToTree(*pathComponents.rbegin(), oid, - archive_entry_mode(entry) & S_IXUSR - ? GIT_FILEMODE_BLOB_EXECUTABLE - : GIT_FILEMODE_BLOB); - - break; - } - - case AE_IFLNK: { - auto target = archive_entry_symlink(entry); - - git_oid oid; - if (git_blob_create_from_buffer(&oid, *this, target, strlen(target))) - throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); - - addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); - - break; - } - - default: - throw Error("file '%s' in tarball has unsupported file type", path); - } - } - - updateBuilders({}); - - auto [oid, _name] = popBuilder(); - - return TarballInfo { - .treeHash = toHash(oid), - .lastModified = lastModified - }; - } - - std::vector> getSubmodules(const Hash & rev) override; + std::vector> getSubmodules(const Hash & rev, bool exportIgnore) override; std::string resolveSubmoduleUrl( const std::string & url, @@ -496,7 +348,16 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return true; } - ref getAccessor(const Hash & rev) override; + /** + * A 'GitInputAccessor' with no regard for export-ignore or any other transformations. + */ + ref getRawAccessor(const Hash & rev); + + ref getAccessor(const Hash & rev, bool exportIgnore) override; + + ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override; + + ref getFileSystemObjectSink() override; static int sidebandProgressCallback(const char * str, int len, void * payload) { @@ -525,27 +386,27 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url)); - Remote remote; - - if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) - throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + // TODO: implement git-credential helper support (preferably via libgit2, which as of 2024-01 does not support that) + // then use code that was removed in this commit (see blame) - char * refspecs[] = {(char *) refspec.c_str()}; - git_strarray refspecs2 { - .strings = refspecs, - .count = 1 - }; - - git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; - // FIXME: for some reason, shallow fetching over ssh barfs - // with "could not read from remote repository". - opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; - opts.callbacks.payload = &act; - opts.callbacks.sideband_progress = sidebandProgressCallback; - opts.callbacks.transfer_progress = transferProgressCallback; + auto dir = this->path; + Strings gitArgs; + if (shallow) { + gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; + } + else { + gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--", url, refspec }; + } - if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) - throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + runProgram(RunOptions { + .program = "git", + .searchPath = true, + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + .args = gitArgs, + .input = {}, + .isInteractive = true + }); } void verifyCommit( @@ -580,7 +441,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this .args = { "-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, - "-C", path.abs(), + "-C", path, "verify-commit", rev.gitRev() }, @@ -608,7 +469,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this Hash treeHashToNarHash(const Hash & treeHash) override { - auto accessor = getAccessor(treeHash); + auto accessor = getAccessor(treeHash, false); // FIXME: exportIgnore fetchers::Attrs cacheKey({{"_what", "treeHashToNarHash"}, {"treeHash", treeHash.gitRev()}}); @@ -623,11 +484,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this } }; -ref GitRepo::openRepo(const CanonPath & path, bool create, bool bare) +ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) { return make_ref(path, create, bare); } +/** + * Raw git tree input accessor. + */ struct GitInputAccessor : InputAccessor { ref repo; @@ -731,20 +595,61 @@ struct GitInputAccessor : InputAccessor /* Recursively look up 'path' relative to the root. */ git_tree_entry * lookup(const CanonPath & path) { - if (path.isRoot()) return nullptr; - auto i = lookupCache.find(path); - if (i == lookupCache.end()) { - TreeEntry entry; - if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) { - if (err != GIT_ENOTFOUND) - throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); - } + if (i != lookupCache.end()) return i->second.get(); + + auto parent = path.parent(); + if (!parent) return nullptr; + + auto name = path.baseName().value(); + + auto parentTree = lookupTree(*parent); + if (!parentTree) return nullptr; + + auto count = git_tree_entrycount(parentTree->get()); + + git_tree_entry * res = nullptr; + + /* Add all the tree entries to the cache to speed up + subsequent lookups. */ + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(parentTree->get(), n); + + TreeEntry copy; + if (git_tree_entry_dup(Setter(copy), entry)) + throw Error("dupping tree entry: %s", git_error_last()->message); + + auto entryName = std::string_view(git_tree_entry_name(entry)); - i = lookupCache.emplace(path, std::move(entry)).first; + if (entryName == name) + res = copy.get(); + + auto path2 = *parent; + path2.push(entryName); + lookupCache.emplace(path2, std::move(copy)).first->second.get(); } - return &*i->second; + return res; + } + + std::optional lookupTree(const CanonPath & path) + { + if (path.isRoot()) { + Tree tree; + if (git_tree_dup(Setter(tree), root.get())) + throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); + return tree; + } + + auto entry = lookup(path); + if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_TREE) + return std::nullopt; + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; } git_tree_entry * need(const CanonPath & path) @@ -816,17 +721,267 @@ struct GitInputAccessor : InputAccessor } }; -ref GitRepoImpl::getAccessor(const Hash & rev) +struct GitExportIgnoreInputAccessor : CachingFilteringInputAccessor { + ref repo; + std::optional rev; + + GitExportIgnoreInputAccessor(ref repo, ref next, std::optional rev) + : CachingFilteringInputAccessor(next, [&](const CanonPath & path) { + return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path)); + }) + , repo(repo) + , rev(rev) + { } + + bool gitAttrGet(const CanonPath & path, const char * attrName, const char * & valueOut) + { + const char * pathCStr = path.rel_c_str(); + + if (rev) { + git_attr_options opts = GIT_ATTR_OPTIONS_INIT; + opts.attr_commit_id = hashToOID(*rev); + // TODO: test that gitattributes from global and system are not used + // (ie more or less: home and etc - both of them!) + opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM; + return git_attr_get_ext( + &valueOut, + *repo, + &opts, + pathCStr, + attrName + ); + } + else { + return git_attr_get( + &valueOut, + *repo, + GIT_ATTR_CHECK_INDEX_ONLY | GIT_ATTR_CHECK_NO_SYSTEM, + pathCStr, + attrName); + } + } + + bool isExportIgnored(const CanonPath & path) + { + const char *exportIgnoreEntry = nullptr; + + // GIT_ATTR_CHECK_INDEX_ONLY: + // > It will use index only for creating archives or for a bare repo + // > (if an index has been specified for the bare repo). + // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48 + if (gitAttrGet(path, "export-ignore", exportIgnoreEntry)) { + if (git_error_last()->klass == GIT_ENOTFOUND) + return false; + else + throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); + } + else { + // Official git will silently reject export-ignore lines that have + // values. We do the same. + return GIT_ATTR_IS_TRUE(exportIgnoreEntry); + } + } + + bool isAllowedUncached(const CanonPath & path) override + { + return !isExportIgnored(path); + } + +}; + +struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink +{ + ref repo; + + struct PendingDir + { + std::string name; + TreeBuilder builder; + }; + + std::vector pendingDirs; + + size_t componentsToStrip = 1; + + void pushBuilder(std::string name) + { + git_treebuilder * b; + if (git_treebuilder_new(&b, *repo, nullptr)) + throw Error("creating a tree builder: %s", git_error_last()->message); + pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) }); + }; + + GitFileSystemObjectSinkImpl(ref repo) : repo(repo) + { + pushBuilder(""); + } + + std::pair popBuilder() + { + assert(!pendingDirs.empty()); + auto pending = std::move(pendingDirs.back()); + git_oid oid; + if (git_treebuilder_write(&oid, pending.builder.get())) + throw Error("creating a tree object: %s", git_error_last()->message); + pendingDirs.pop_back(); + return {oid, pending.name}; + }; + + void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode) + { + assert(!pendingDirs.empty()); + auto & pending = pendingDirs.back(); + if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode)) + throw Error("adding a file to a tree builder: %s", git_error_last()->message); + }; + + void updateBuilders(std::span names) + { + // Find the common prefix of pendingDirs and names. + size_t prefixLen = 0; + for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen) + if (names[prefixLen] != pendingDirs[prefixLen + 1].name) + break; + + // Finish the builders that are not part of the common prefix. + for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) { + auto [oid, name] = popBuilder(); + addToTree(name, oid, GIT_FILEMODE_TREE); + } + + // Create builders for the new directories. + for (auto n = prefixLen; n < names.size(); ++n) + pushBuilder(names[n]); + }; + + bool prepareDirs(const std::vector & pathComponents, bool isDir) + { + std::span pathComponents2{pathComponents}; + + if (pathComponents2.size() <= componentsToStrip) return false; + pathComponents2 = pathComponents2.subspan(componentsToStrip); + + updateBuilders( + isDir + ? pathComponents2 + : pathComponents2.first(pathComponents2.size() - 1)); + + return true; + } + + void createRegularFile( + const Path & path, + std::function func) override + { + auto pathComponents = tokenizeString>(path, "/"); + if (!prepareDirs(pathComponents, false)) return; + + git_writestream * stream = nullptr; + if (git_blob_create_from_stream(&stream, *repo, nullptr)) + throw Error("creating a blob stream object: %s", git_error_last()->message); + + struct CRF : CreateRegularFileSink { + const Path & path; + GitFileSystemObjectSinkImpl & back; + git_writestream * stream; + bool executable = false; + CRF(const Path & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) + : path(path), back(back), stream(stream) + {} + void operator () (std::string_view data) override + { + if (stream->write(stream, data.data(), data.size())) + throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + } + void isExecutable() override + { + executable = true; + } + } crf { path, *this, stream }; + func(crf); + + git_oid oid; + if (git_blob_create_from_stream_commit(&oid, stream)) + throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message); + + addToTree(*pathComponents.rbegin(), oid, + crf.executable + ? GIT_FILEMODE_BLOB_EXECUTABLE + : GIT_FILEMODE_BLOB); + } + + void createDirectory(const Path & path) override + { + auto pathComponents = tokenizeString>(path, "/"); + (void) prepareDirs(pathComponents, true); + } + + void createSymlink(const Path & path, const std::string & target) override + { + auto pathComponents = tokenizeString>(path, "/"); + if (!prepareDirs(pathComponents, false)) return; + + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) + throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); + + addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); + } + + Hash sync() override { + updateBuilders({}); + + auto [oid, _name] = popBuilder(); + + return toHash(oid); + } +}; + +ref GitRepoImpl::getRawAccessor(const Hash & rev) { - return make_ref(ref(shared_from_this()), rev); + auto self = ref(shared_from_this()); + return make_ref(self, rev); } -std::vector> GitRepoImpl::getSubmodules(const Hash & rev) +ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore) +{ + auto self = ref(shared_from_this()); + ref rawGitAccessor = getRawAccessor(rev); + if (exportIgnore) { + return make_ref(self, rawGitAccessor, rev); + } + else { + return rawGitAccessor; + } +} + +ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) +{ + auto self = ref(shared_from_this()); + ref fileAccessor = + AllowListInputAccessor::create( + makeFSInputAccessor(path), + std::set { wd.files }, + std::move(makeNotAllowedError)); + if (exportIgnore) { + return make_ref(self, fileAccessor, std::nullopt); + } + else { + return fileAccessor; + } +} + +ref GitRepoImpl::getFileSystemObjectSink() +{ + return make_ref(ref(shared_from_this())); +} + +std::vector> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore) { /* Read the .gitmodules files from this revision. */ CanonPath modulesFile(".gitmodules"); - auto accessor = getAccessor(rev); + auto accessor = getAccessor(rev, exportIgnore); if (!accessor->pathExists(modulesFile)) return {}; /* Parse it and get the revision of each submodule. */ @@ -837,8 +992,10 @@ std::vector> GitRepoImpl::getSubmodules std::vector> result; - for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) { - auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(submodule.path); + auto rawAccessor = getRawAccessor(rev); + + for (auto & submodule : parseSubmodules(pathTemp)) { + auto rev = rawAccessor->getSubmoduleRev(submodule.path); result.push_back({std::move(submodule), rev}); } @@ -847,9 +1004,9 @@ std::vector> GitRepoImpl::getSubmodules ref getTarballCache() { - static CanonPath repoDir(getCacheDir() + "/nix/tarball-cache"); + static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache"; - return make_ref(repoDir, true, true); + return GitRepo::openRepo(repoDir, true, true); } } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index b8b31530ada..fbb2d947b45 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -1,17 +1,27 @@ #pragma once +#include "filtering-input-accessor.hh" #include "input-accessor.hh" +#include "fs-sink.hh" namespace nix { namespace fetchers { struct PublicKey; } +struct GitFileSystemObjectSink : FileSystemObjectSink +{ + /** + * Flush builder and return a final Git hash. + */ + virtual Hash sync() = 0; +}; + struct GitRepo { virtual ~GitRepo() { } - static ref openRepo(const CanonPath & path, bool create = false, bool bare = false); + static ref openRepo(const std::filesystem::path & path, bool create = false, bool bare = false); virtual uint64_t getRevCount(const Hash & rev) = 0; @@ -57,23 +67,19 @@ struct GitRepo * Return the submodules of this repo at the indicated revision, * along with the revision of each submodule. */ - virtual std::vector> getSubmodules(const Hash & rev) = 0; + virtual std::vector> getSubmodules(const Hash & rev, bool exportIgnore) = 0; virtual std::string resolveSubmoduleUrl( const std::string & url, const std::string & base) = 0; - struct TarballInfo - { - Hash treeHash; - time_t lastModified; - }; + virtual bool hasObject(const Hash & oid) = 0; - virtual TarballInfo importTarball(Source & source) = 0; + virtual ref getAccessor(const Hash & rev, bool exportIgnore) = 0; - virtual bool hasObject(const Hash & oid) = 0; + virtual ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0; - virtual ref getAccessor(const Hash & rev) = 0; + virtual ref getFileSystemObjectSink() = 0; virtual void fetch( const std::string & url, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 189cbee0b1a..df90b01a66c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "fetchers.hh" #include "users.hh" #include "cache.hh" @@ -9,7 +10,6 @@ #include "processes.hh" #include "git.hh" #include "fs-input-accessor.hh" -#include "filtering-input-accessor.hh" #include "mounted-input-accessor.hh" #include "git-utils.hh" #include "logging.hh" @@ -50,10 +50,12 @@ bool touchCacheFile(const Path & path, time_t touch_time) return lutimes(path.c_str(), times) == 0; } -Path getCachePath(std::string_view key) +Path getCachePath(std::string_view key, bool shallow) { - return getCacheDir() + "/nix/gitv3/" + - hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false); + return getCacheDir() + + "/nix/gitv3/" + + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + + (shallow ? "-shallow" : ""); } // Returns the name of the HEAD branch. @@ -92,7 +94,8 @@ std::optional readHead(const Path & path) // Persist the HEAD ref from the remote repo in the local cached repo. bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { - Path cacheDir = getCachePath(actualUrl); + // set shallow=false as HEAD will never be queried for a shallow repo + Path cacheDir = getCachePath(actualUrl, false); try { runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { @@ -107,7 +110,8 @@ std::optional readHeadCached(const std::string & actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. - Path cacheDir = getCachePath(actualUrl); + // set shallow=false as HEAD will never be queried for a shallow repo + Path cacheDir = getCachePath(actualUrl, false); Path headRefFile = cacheDir + "/HEAD"; time_t now = time(0); @@ -174,7 +178,7 @@ struct GitInputScheme : InputScheme for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit") + else if (name == "shallow" || name == "submodules" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -199,6 +203,7 @@ struct GitInputScheme : InputScheme "rev", "shallow", "submodules", + "exportIgnore", "lastModified", "revCount", "narHash", @@ -250,6 +255,8 @@ struct GitInputScheme : InputScheme url.query.insert_or_assign("shallow", "1"); if (getSubmodulesAttr(input)) url.query.insert_or_assign("submodules", "1"); + if (maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false)) + url.query.insert_or_assign("exportIgnore", "1"); if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) url.query.insert_or_assign("verifyCommit", "1"); auto publicKeys = getPublicKeys(input.attrs); @@ -305,17 +312,28 @@ struct GitInputScheme : InputScheme if (!repoInfo.isLocal) throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); - writeFile((CanonPath(repoInfo.url) + path).abs(), contents); + writeFile((CanonPath(repoInfo.url) / path).abs(), contents); - runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + auto result = runProgram(RunOptions { + .program = "git", + .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, + }); + auto exitCode = WEXITSTATUS(result.first); - // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` - logger->pause(); - Finally restoreLogger([]() { logger->resume(); }); - if (commitMsg) + if (exitCode != 0) { + // The path is not `.gitignore`d, we can add the file. runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + + + if (commitMsg) { + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); + runProgram("git", true, + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); + } + } } struct RepoInfo @@ -354,6 +372,11 @@ struct GitInputScheme : InputScheme return maybeGetBoolAttr(input.attrs, "submodules").value_or(false); } + bool getExportIgnoreAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false); + } + bool getAllRefsAttr(const Input & input) const { return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); @@ -361,14 +384,14 @@ struct GitInputScheme : InputScheme RepoInfo getRepoInfo(const Input & input) const { - auto checkHashType = [&](const std::optional & hash) + auto checkHashAlgorithm = [&](const std::optional & hash) { if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; if (auto rev = input.getRev()) - checkHashType(rev); + checkHashAlgorithm(rev); RepoInfo repoInfo; @@ -385,7 +408,7 @@ struct GitInputScheme : InputScheme // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(); + repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); return repoInfo; } @@ -399,7 +422,7 @@ struct GitInputScheme : InputScheme if (auto res = cache->lookup(key)) return getIntAttr(*res, "lastModified"); - auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev); + auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev); cache->upsert(key, Attrs{{"lastModified", lastModified}}); @@ -417,7 +440,7 @@ struct GitInputScheme : InputScheme Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); - auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev); + auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); cache->upsert(key, Attrs{{"revCount", revCount}}); @@ -427,7 +450,7 @@ struct GitInputScheme : InputScheme std::string getDefaultRef(const RepoInfo & repoInfo) const { auto head = repoInfo.isLocal - ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef() + ? GitRepo::openRepo(repoInfo.url)->getWorkdirRef() : readHeadCached(repoInfo.url); if (!head) { warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); @@ -480,16 +503,16 @@ struct GitInputScheme : InputScheme if (repoInfo.isLocal) { repoDir = repoInfo.url; if (!input.getRev()) - input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); + input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { - Path cacheDir = getCachePath(repoInfo.url); + Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input)); repoDir = cacheDir; repoInfo.gitDir = "."; createDirs(dirOf(cacheDir)); PathLocks cacheDirLock({cacheDir}); - auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true); + auto repo = GitRepo::openRepo(cacheDir, true, true); Path localRefFile = ref.compare(0, 5, "refs/") == 0 @@ -558,7 +581,7 @@ struct GitInputScheme : InputScheme // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - auto repo = GitRepo::openRepo(CanonPath(repoDir)); + auto repo = GitRepo::openRepo(repoDir); auto isShallow = repo->isShallow(); @@ -582,7 +605,8 @@ struct GitInputScheme : InputScheme verifyCommit(input, repo); - auto accessor = repo->getAccessor(rev); + bool exportIgnore = getExportIgnoreAttr(input); + auto accessor = repo->getAccessor(rev, exportIgnore); accessor->setPathDisplay("«" + input.to_string() + "»"); @@ -592,7 +616,7 @@ struct GitInputScheme : InputScheme if (getSubmodulesAttr(input)) { std::map> mounts; - for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) { auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url); debug("Git submodule %s: %s %s %s -> %s", submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); @@ -602,6 +626,7 @@ struct GitInputScheme : InputScheme if (submodule.branch != "") attrs.insert_or_assign("ref", submodule.branch); attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); @@ -632,10 +657,13 @@ struct GitInputScheme : InputScheme for (auto & submodule : repoInfo.workdirInfo.submodules) repoInfo.workdirInfo.files.insert(submodule.path); + auto repo = GitRepo::openRepo(repoInfo.url, false, false); + + auto exportIgnore = getExportIgnoreAttr(input); + ref accessor = - AllowListInputAccessor::create( - makeFSInputAccessor(CanonPath(repoInfo.url)), - std::move(repoInfo.workdirInfo.files), + repo->getAccessor(repoInfo.workdirInfo, + exportIgnore, makeNotAllowedError(repoInfo.url)); /* If the repo has submodules, return a mounted input accessor @@ -645,10 +673,12 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & submodule : repoInfo.workdirInfo.submodules) { - auto submodulePath = CanonPath(repoInfo.url) + submodule.path; + auto submodulePath = CanonPath(repoInfo.url) / submodule.path; fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); attrs.insert_or_assign("url", submodulePath.abs()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); @@ -666,7 +696,7 @@ struct GitInputScheme : InputScheme } if (!repoInfo.workdirInfo.isDirty) { - auto repo = GitRepo::openRepo(CanonPath(repoInfo.url)); + auto repo = GitRepo::openRepo(repoInfo.url); if (auto ref = repo->getWorkdirRef()) input.attrs.insert_or_assign("ref", *ref); @@ -705,6 +735,16 @@ struct GitInputScheme : InputScheme auto repoInfo = getRepoInfo(input); + if (getExportIgnoreAttr(input) + && getSubmodulesAttr(input)) { + /* In this situation, we don't have a git CLI behavior that we can copy. + `git archive` does not support submodules, so it is unclear whether + rules from the parent should affect the submodule or not. + When git may eventually implement this, we need Nix to match its + behavior. */ + throw UnimplementedError("exportIgnore and submodules are not supported together yet"); + } + auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.isLocal ? getAccessorFromCommit(store, repoInfo, std::move(input)) @@ -718,7 +758,7 @@ struct GitInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { if (auto rev = input.getRev()) - return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : ""); + return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); else return std::nullopt; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index e9cc23d0916..d204ae76c36 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -9,6 +9,7 @@ #include "fetch-settings.hh" #include "input-accessor.hh" #include "tarball.hh" +#include "tarfile.hh" #include "git-utils.hh" #include @@ -212,7 +213,13 @@ struct GitArchiveInputScheme : InputScheme virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair downloadArchive(ref store, Input input) const + struct TarballInfo + { + Hash treeHash; + time_t lastModified; + }; + + std::pair downloadArchive(ref store, Input input) const { if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); @@ -239,7 +246,7 @@ struct GitArchiveInputScheme : InputScheme auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); if (getTarballCache()->hasObject(treeHash)) - return {std::move(input), GitRepo::TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; + return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); } @@ -254,17 +261,26 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); - auto tarballInfo = getTarballCache()->importTarball(*source); + TarArchive archive { *source }; + auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); + + TarballInfo tarballInfo { + .treeHash = parseSink->sync(), + .lastModified = lastModified + }; cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); + #if 0 if (upstreamTreeHash != tarballInfo.treeHash) warn( "Git tree hash mismatch for revision '%s' of '%s': " "expected '%s', got '%s'. " "This can happen if the Git repository uses submodules.", rev->gitRev(), input.to_string(), upstreamTreeHash->gitRev(), tarballInfo.treeHash.gitRev()); + #endif return {std::move(input), tarballInfo}; } @@ -276,10 +292,12 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); - auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash); + auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false); accessor->setPathDisplay("«" + input.to_string() + "»"); + accessor->fingerprint = input.getFingerprint(store); + return {accessor, input}; } @@ -348,6 +366,7 @@ struct GitHubInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); + return RefInfo { .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), .treeHash = Hash::parseAny(std::string { json["commit"]["tree"]["sha"] }, HashAlgorithm::SHA1) @@ -421,6 +440,7 @@ struct GitLabInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); + return RefInfo { .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) }; diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc deleted file mode 100644 index 39b46e684c3..00000000000 --- a/src/libfetchers/input-accessor.cc +++ /dev/null @@ -1,119 +0,0 @@ -#include "input-accessor.hh" -#include "util.hh" -#include "store-api.hh" -#include "cache.hh" - -namespace nix { - -StorePath InputAccessor::fetchToStore( - ref store, - const CanonPath & path, - std::string_view name, - FileIngestionMethod method, - PathFilter * filter, - RepairFlag repair) -{ - // FIXME: add an optimisation for the case where the accessor is - // an FSInputAccessor pointing to a store path. - - std::optional cacheKey; - - if (!filter && fingerprint) { - cacheKey = fetchers::Attrs{ - {"_what", "fetchToStore"}, - {"store", store->storeDir}, - {"name", std::string(name)}, - {"fingerprint", *fingerprint}, - {"method", (uint8_t) method}, - {"path", path.abs()} - }; - if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { - debug("store path cache hit for '%s'", showPath(path)); - return res->second; - } - } else - debug("source path '%s' is uncacheable", showPath(path)); - - Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); - - auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(path, sink, filter ? *filter : defaultPathFilter); - else - readFile(path, sink); - }); - - auto storePath = - settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first - : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair); - - if (cacheKey) - fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); - - return storePath; -} - -std::ostream & operator << (std::ostream & str, const SourcePath & path) -{ - str << path.to_string(); - return str; -} - -StorePath SourcePath::fetchToStore( - ref store, - std::string_view name, - FileIngestionMethod method, - PathFilter * filter, - RepairFlag repair) const -{ - return accessor->fetchToStore(store, path, name, method, filter, repair); -} - -std::string_view SourcePath::baseName() const -{ - return path.baseName().value_or("source"); -} - -SourcePath SourcePath::parent() const -{ - auto p = path.parent(); - assert(p); - return {accessor, std::move(*p)}; -} - -SourcePath SourcePath::resolveSymlinks() const -{ - auto res = SourcePath(accessor); - - int linksAllowed = 1024; - - std::list todo; - for (auto & c : path) - todo.push_back(std::string(c)); - - while (!todo.empty()) { - auto c = *todo.begin(); - todo.pop_front(); - if (c == "" || c == ".") - ; - else if (c == "..") - res.path.pop(); - else { - res.path.push(c); - if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { - if (!linksAllowed--) - throw Error("infinite symlink recursion in path '%s'", path); - auto target = res.readLink(); - res.path.pop(); - if (hasPrefix(target, "/")) - res.path = CanonPath::root; - todo.splice(todo.begin(), tokenizeString>(target, "/")); - } - } - } - - return res; -} - -} diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh deleted file mode 100644 index 340dcd14625..00000000000 --- a/src/libfetchers/input-accessor.hh +++ /dev/null @@ -1,181 +0,0 @@ -#pragma once -///@file - -#include "source-accessor.hh" -#include "ref.hh" -#include "types.hh" -#include "file-system.hh" -#include "repair-flag.hh" -#include "content-address.hh" -#include "hash.hh" - -namespace nix { - -MakeError(RestrictedPathError, Error); - -struct SourcePath; -class StorePath; -class Store; - -struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this -{ - std::optional fingerprint; - - /** - * Return the maximum last-modified time of the files in this - * tree, if available. - */ - virtual std::optional getLastModified() - { - return std::nullopt; - } - - StorePath fetchToStore( - ref store, - const CanonPath & path, - std::string_view name = "source", - FileIngestionMethod method = FileIngestionMethod::Recursive, - PathFilter * filter = nullptr, - RepairFlag repair = NoRepair); -}; - -typedef std::function MakeNotAllowedError; - -ref makePatchingInputAccessor( - ref next, - const std::vector & patches); - -/** - * An abstraction for accessing source files during - * evaluation. Currently, it's just a wrapper around `CanonPath` that - * accesses files in the regular filesystem, but in the future it will - * support fetching files in other ways. - */ -struct SourcePath -{ - ref accessor; - CanonPath path; - - SourcePath(ref accessor, CanonPath path = CanonPath::root) - : accessor(std::move(accessor)) - , path(std::move(path)) - { } - - std::string_view baseName() const; - - /** - * Construct the parent of this `SourcePath`. Aborts if `this` - * denotes the root. - */ - SourcePath parent() const; - - /** - * If this `SourcePath` denotes a regular file (not a symlink), - * return its contents; otherwise throw an error. - */ - std::string readFile() const - { return accessor->readFile(path); } - - /** - * Return whether this `SourcePath` denotes a file (of any type) - * that exists - */ - bool pathExists() const - { return accessor->pathExists(path); } - - /** - * Return stats about this `SourcePath`, or throw an exception if - * it doesn't exist. - */ - InputAccessor::Stat lstat() const - { return accessor->lstat(path); } - - /** - * Return stats about this `SourcePath`, or std::nullopt if it - * doesn't exist. - */ - std::optional maybeLstat() const - { return accessor->maybeLstat(path); } - - /** - * If this `SourcePath` denotes a directory (not a symlink), - * return its directory entries; otherwise throw an error. - */ - InputAccessor::DirEntries readDirectory() const - { return accessor->readDirectory(path); } - - /** - * If this `SourcePath` denotes a symlink, return its target; - * otherwise throw an error. - */ - std::string readLink() const - { return accessor->readLink(path); } - - /** - * Dump this `SourcePath` to `sink` as a NAR archive. - */ - void dumpPath( - Sink & sink, - PathFilter & filter = defaultPathFilter) const - { return accessor->dumpPath(path, sink, filter); } - - /** - * Copy this `SourcePath` to the Nix store. - */ - StorePath fetchToStore( - ref store, - std::string_view name = "source", - FileIngestionMethod method = FileIngestionMethod::Recursive, - PathFilter * filter = nullptr, - RepairFlag repair = NoRepair) const; - - /** - * Return the location of this path in the "real" filesystem, if - * it has a physical location. - */ - std::optional getPhysicalPath() const - { return accessor->getPhysicalPath(path); } - - std::string to_string() const - { return accessor->showPath(path); } - - /** - * Append a `CanonPath` to this path. - */ - SourcePath operator + (const CanonPath & x) const - { return {accessor, path + x}; } - - /** - * Append a single component `c` to this path. `c` must not - * contain a slash. A slash is implicitly added between this path - * and `c`. - */ - SourcePath operator + (std::string_view c) const - { return {accessor, path + c}; } - - bool operator == (const SourcePath & x) const - { - return std::tie(accessor, path) == std::tie(x.accessor, x.path); - } - - bool operator != (const SourcePath & x) const - { - return std::tie(accessor, path) != std::tie(x.accessor, x.path); - } - - bool operator < (const SourcePath & x) const - { - return std::tie(accessor, path) < std::tie(x.accessor, x.path); - } - - /** - * Resolve any symlinks in this `SourcePath` (including its - * parents). The result is a `SourcePath` in which no element is a - * symlink. - */ - SourcePath resolveSymlinks() const; -}; - -std::ostream & operator << (std::ostream & str, const SourcePath & path); - -} diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 266e7a21149..e54db4937cc 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive +libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive libfetchers_LIBS = libutil libstore diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 057f3e37f73..88a2e34e8d1 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -1,5 +1,6 @@ #include "memory-input-accessor.hh" #include "memory-source-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh index b75b02bfd61..508b07722ef 100644 --- a/src/libfetchers/memory-input-accessor.hh +++ b/src/libfetchers/memory-input-accessor.hh @@ -1,4 +1,5 @@ #include "input-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 55bab2663aa..dc57de5b9c5 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -7,6 +7,7 @@ #include "store-api.hh" #include "url-parts.hh" #include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" #include "fetch-settings.hh" #include @@ -132,7 +133,7 @@ struct MercurialInputScheme : InputScheme if (!isLocal) throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); - auto absPath = CanonPath(repoPath) + path; + auto absPath = CanonPath(repoPath) / path; writeFile(absPath.abs(), contents); @@ -202,7 +203,12 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter); + PosixSourceAccessor accessor; + auto storePath = store->addToStore( + input.getName(), + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, + filter); return storePath; } @@ -210,7 +216,7 @@ struct MercurialInputScheme : InputScheme if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); - auto checkHashType = [&](const std::optional & hash) + auto checkHashAlgorithm = [&](const std::optional & hash) { if (hash.has_value() && hash->algo != HashAlgorithm::SHA1) throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); @@ -219,7 +225,7 @@ struct MercurialInputScheme : InputScheme auto getLockedAttrs = [&]() { - checkHashType(input.getRev()); + checkHashAlgorithm(input.getRev()); return Attrs({ {"type", "hg"}, @@ -237,7 +243,7 @@ struct MercurialInputScheme : InputScheme }; if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) + if (auto res = getCache()->lookup(*store, getLockedAttrs())) return makeResult(res->first, std::move(res->second)); } @@ -250,7 +256,7 @@ struct MercurialInputScheme : InputScheme {"ref", *input.getRef()}, }); - if (auto res = getCache()->lookup(store, unlockedAttrs)) { + if (auto res = getCache()->lookup(*store, unlockedAttrs)) { auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); @@ -296,7 +302,7 @@ struct MercurialInputScheme : InputScheme auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); - if (auto res = getCache()->lookup(store, getLockedAttrs())) + if (auto res = getCache()->lookup(*store, getLockedAttrs())) return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); @@ -306,7 +312,8 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); - auto storePath = store->addToStore(name, tmpDir); + PosixSourceAccessor accessor; + auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir }); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -315,14 +322,14 @@ struct MercurialInputScheme : InputScheme if (!origRev) getCache()->add( - store, + *store, unlockedAttrs, infoAttrs, storePath, false); getCache()->add( - store, + *store, getLockedAttrs(), infoAttrs, storePath, diff --git a/src/libfetchers/patching-input-accessor.hh b/src/libfetchers/patching-input-accessor.hh new file mode 100644 index 00000000000..c5e741ef1b6 --- /dev/null +++ b/src/libfetchers/patching-input-accessor.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +ref makePatchingInputAccessor( + ref next, + const std::vector & patches); + +} diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 5409b3ffe8f..1f555dabf08 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -2,6 +2,7 @@ #include "store-api.hh" #include "archive.hh" #include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" namespace nix::fetchers { @@ -105,7 +106,7 @@ struct PathInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); + writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); } CanonPath getAbsPath(const Input & input) const @@ -130,7 +131,8 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != input.getName() || !store->isValidPath(*storePath)) { Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", absPath)); - storePath = store->addToStore(input.getName(), absPath.abs()); + PosixSourceAccessor accessor; + storePath = store->addToStore(input.getName(), accessor, absPath); auto narHash = store->queryPathInfo(*storePath)->narHash; input2.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); } else @@ -150,7 +152,7 @@ struct PathInputScheme : InputScheme return {makeStorePathAccessor(store, *storePath), std::move(input2)}; } else { - return {makeFSInputAccessor(absPath), std::move(input2)}; + return {makeFSInputAccessor(std::filesystem::path(absPath.abs())), std::move(input2)}; } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index c05c4cdee5f..a1a33625760 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -8,6 +8,7 @@ #include "tarfile.hh" #include "types.hh" #include "split.hh" +#include "posix-source-accessor.hh" #include "fs-input-accessor.hh" #include "store-api.hh" #include "git-utils.hh" @@ -29,7 +30,7 @@ DownloadFileResult downloadFile( {"name", name}, }); - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupExpired(*store, inAttrs); auto useCached = [&]() -> DownloadFileResult { @@ -96,7 +97,7 @@ DownloadFileResult downloadFile( inAttrs.insert_or_assign("url", url); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); getCache()->add( - store, + *store, inAttrs, infoAttrs, *storePath, @@ -131,7 +132,7 @@ DownloadTarballResult downloadTarball( .treeHash = treeHash, .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), - .accessor = getTarballCache()->getAccessor(treeHash), + .accessor = getTarballCache()->getAccessor(treeHash, false), }; }; @@ -157,7 +158,9 @@ DownloadTarballResult downloadTarball( /* Note: if the download is cached, `importTarball()` will receive no data, which causes it to import an empty tarball. */ - auto tarballInfo = getTarballCache()->importTarball(*source); + TarArchive archive { *source }; + auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); auto res(_res->lock()); @@ -167,8 +170,8 @@ DownloadTarballResult downloadTarball( infoAttrs = cached->infoAttrs; } else { infoAttrs.insert_or_assign("etag", res->etag); - infoAttrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); - infoAttrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); + infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 862ef355b95..7b9b3c5b587 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -340,7 +340,7 @@ int handleExceptions(const std::string & programName, std::function fun) return 1; } catch (BaseError & e) { logError(e.info()); - return e.status; + return e.info().status; } catch (std::bad_alloc & e) { printError(error + "out of memory"); return 1; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2837e893491..189d1d305cf 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -12,6 +12,7 @@ #include "thread-pool.hh" #include "callback.hh" #include "signals.hh" +#include "archive.hh" #include #include @@ -27,7 +28,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) , Store(params) { if (secretKeyFile != "") - secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); + signer = std::make_unique( + SecretKey { readFile(secretKeyFile) }); StringSink sink; sink << narVersionMagic1; @@ -233,14 +235,14 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex2("^[0-9a-f]{38}\\.debug$"); for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + s1; + auto dir = buildIdDir / s1; if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { - auto debugPath = dir + s2; + auto debugPath = dir / s2; if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) @@ -273,7 +275,7 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ - if (secretKey) narInfo->sign(*this, *secretKey); + if (signer) narInfo->sign(*this, *signer); writeNarInfo(narInfo); @@ -300,24 +302,60 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource }}); } -StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath BinaryCacheStore::addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { - if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) - unsupported("addToStoreFromDump"); - return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { + std::optional caHash; + std::string nar; + + if (auto * dump2p = dynamic_cast(&dump)) { + auto & dump2 = *dump2p; + // Hack, this gives us a "replayable" source so we can compute + // multiple hashes more easily. + caHash = hashString(HashAlgorithm::SHA256, dump2.s); + switch (method.getFileIngestionMethod()) { + case FileIngestionMethod::Recursive: + // The dump is already NAR in this case, just use it. + nar = dump2.s; + break; + case FileIngestionMethod::Flat: + // The dump is Flat, so we need to convert it to NAR with a + // single file. + StringSink s; + dumpString(dump2.s, s); + nar = std::move(s.s); + break; + } + } else { + // Otherwise, we have to do th same hashing as NAR so our single + // hash will suffice for both purposes. + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + unsupported("addToStoreFromDump"); + } + StringSource narDump { nar }; + + // Use `narDump` if we wrote to `nar`. + Source & narDump2 = nar.size() > 0 + ? static_cast(narDump) + : dump; + + return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = nar.first, - .references = { + ContentAddressWithReferences::fromParts( + method, + caHash ? *caHash : nar.first, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }, + }), nar.first, }; info.narSize = nar.second; @@ -399,72 +437,36 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - HashSink sink { hashAlgo }; - if (method == FileIngestionMethod::Recursive) { - dumpPath(srcPath, sink, filter); - } else { - readFile(srcPath, sink); - } - auto h = sink.finish().first; + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink, filter); + accessor.dumpPath(path, sink, filter); }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = h, - .references = { + ContentAddressWithReferences::fromParts( + method, + h, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }, - nar.first, - }; - info.narSize = nar.second; - return info; - })->path; -} - -StorePath BinaryCacheStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - auto textHash = hashString(HashAlgorithm::SHA256, s); - auto path = makeTextPath(name, TextInfo { { textHash }, references }); - - if (!repair && isValidPath(path)) - return path; - - StringSink sink; - dumpString(s, sink); - StringSource source(sink.s); - return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { - *this, - std::string { name }, - TextInfo { - .hash = textHash, - .references = references, - }, + }), nar.first, }; info.narSize = nar.second; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 395e1b47984..00ab73905e2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "crypto.hh" +#include "signature/local-keys.hh" #include "store-api.hh" #include "log-store.hh" @@ -57,8 +57,7 @@ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, { private: - - std::unique_ptr secretKey; + std::unique_ptr signer; protected: @@ -123,22 +122,22 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; - - StorePath addTextToStore( std::string_view name, - std::string_view s, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, const StorePathSet & references, + PathFilter & filter, RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f8728ed4a68..29bf2852f80 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -223,7 +223,7 @@ void DerivationGoal::haveDerivation() if (!drv->type().hasKnownOutputPaths()) experimentalFeatureSettings.require(Xp::CaDerivations); - if (!drv->type().isPure()) { + if (drv->type().isImpure()) { experimentalFeatureSettings.require(Xp::ImpureDerivations); for (auto & [outputName, output] : drv->outputs) { @@ -304,7 +304,7 @@ void DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { done(BuildResult::TransientFailure, {}, @@ -397,9 +397,9 @@ void DerivationGoal::gaveUpOnSubstitution() for (const auto & [inputDrvPath, inputNode] : dynamic_cast(drv.get())->inputDrvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (!inputDrv.type().isPure()) + if (inputDrv.type().isImpure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), worker.store.printStorePath(inputDrvPath)); @@ -439,7 +439,7 @@ void DerivationGoal::gaveUpOnSubstitution() void DerivationGoal::repairClosure() { - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); /* If we're repairing, we now know that our own outputs are valid. Now check whether the other paths in the outputs closure are @@ -708,7 +708,7 @@ void DerivationGoal::tryToBuild() if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); + fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } @@ -762,7 +762,7 @@ void DerivationGoal::tryToBuild() the wake-up timeout expires. */ if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -780,9 +780,13 @@ void DerivationGoal::tryToBuild() void DerivationGoal::tryLocalBuild() { throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + R"( + Unable to build with a primary store that isn't a local store; + either pass a different '--store' or enable remote builds. + + For more information check 'man nix.conf' and search for '/machines'. + )" + ); } @@ -891,7 +895,7 @@ void runPostBuildHook( if (hook == "") return; - Activity act(logger, lvlInfo, actPostBuildHook, + Activity act(logger, lvlTalkative, actPostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{store.printStorePath(drvPath)}); PushActivity pact(act.id); @@ -987,7 +991,7 @@ void DerivationGoal::buildDone() diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), + Magenta(worker.store.printStorePath(drvPath)), statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { @@ -1100,7 +1104,7 @@ void DerivationGoal::resolvedFinished() worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); }(); - if (drv->type().isPure()) { + if (!drv->type().isImpure()) { auto newRealisation = realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; newRealisation.signatures.clear(); @@ -1395,7 +1399,7 @@ void DerivationGoal::flushLine() std::map> DerivationGoal::queryPartialDerivationOutputMap() { - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { std::map> res; for (auto & [name, output] : drv->outputs) @@ -1411,7 +1415,7 @@ std::map> DerivationGoal::queryPartialDeri OutputPathMap DerivationGoal::queryDerivationOutputMap() { - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) @@ -1428,7 +1432,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() std::pair DerivationGoal::checkPathValidity() { - if (!drv->type().isPure()) return { false, {} }; + if (drv->type().isImpure()) return { false, {} }; bool checkHash = buildMode == bmRepair; auto wantedOutputsLeft = std::visit(overloaded { @@ -1523,7 +1527,7 @@ void DerivationGoal::done( outputLocks.unlock(); buildResult.status = status; if (ex) - buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); + buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg)); if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; if (buildResult.status == BuildResult::PermanentFailure) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 74eca63f316..d4bead28edf 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -26,14 +26,14 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) - failed.insert(std::string { i2->drvPath.to_string() }); + failed.insert(printStorePath(i2->drvPath)); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(std::string { i2->storePath.to_string()}); + failed.insert(printStorePath(i2->storePath)); } } if (failed.size() == 1 && ex) { - ex->status = worker.failingExitStatus(); + ex->withExitStatus(worker.failingExitStatus()); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); @@ -104,7 +104,7 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { - goal->ex->status = worker.failingExitStatus(); + goal->ex->withExitStatus(worker.failingExitStatus()); throw std::move(*goal->ex); } else throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 802b39f8459..b373c74b26a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -20,6 +20,7 @@ #include "child.hh" #include "unix-domain-socket.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -91,7 +92,7 @@ void handleDiffHook( } catch (Error & error) { ErrorInfo ei = error.info(); // FIXME: wrap errors. - ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); + ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); logError(ei); } } @@ -231,7 +232,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -1290,13 +1291,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In { throw Error("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1306,26 +1308,15 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In goal.addDependency(info.path); } - StorePath addTextToStore( + StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, + ContentAddressMethod method, + HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - StorePath addToStoreFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + RepairFlag repair) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair); goal.addDependency(path); return path; } @@ -1504,7 +1495,7 @@ void LocalDerivationGoal::startDaemon() daemon::processConnection(store, from, to, NotTrusted, daemon::Recursive); debug("terminated daemon connection"); - } catch (SysError &) { + } catch (SystemError &) { ignoreException(); } }); @@ -1716,7 +1707,7 @@ void LocalDerivationGoal::runChild() try { if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } + } catch (SystemError &) { } #if __linux__ if (useChroot) { @@ -2139,16 +2130,17 @@ void LocalDerivationGoal::runChild() try { logger = makeJSONLogger(*logger); - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); + std::map outputs; + for (auto & e : drv->outputs) + outputs.insert_or_assign(e.first, + worker.store.printStorePath(scratchOutputs.at(e.first))); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); + builtinFetchurl(*drv, outputs, netrcData); else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); + builtinBuildenv(*drv, outputs); else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); + builtinUnpackChannel(*drv, outputs); else throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); _exit(0); @@ -2453,8 +2445,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() throw BuildError( "output path %1% without valid stats info", actualPath); - if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || - outputHash.method == ContentAddressMethod { TextIngestionMethod {} }) + if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) @@ -2466,38 +2457,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink {outputHash.hashAlgo, oldHashPart }; - std::visit(overloaded { - [&](const TextIngestionMethod &) { - readFile(actualPath, caSink); - }, - [&](const FileIngestionMethod & m2) { - switch (m2) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - }, - }, outputHash.method.raw); - auto got = caSink.finish().first; - - auto optCA = ContentAddressWithReferences::fromPartsOpt( - outputHash.method, - std::move(got), - rewriteRefs()); - if (!optCA) { - // TODO track distinct failure modes separately (at the time of - // writing there is just one but `nullopt` is unclear) so this - // message can't get out of sync. - throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing"); - } + auto got = ({ + HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + PosixSourceAccessor accessor; + dumpPath( + accessor, CanonPath { actualPath }, + caSink, + outputHash.method.getFileIngestionMethod()); + caSink.finish().first; + }); + ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - std::move(*optCA), + ContentAddressWithReferences::fromParts( + outputHash.method, + std::move(got), + rewriteRefs()), Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2511,9 +2487,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); - newInfo0.narHash = narHashAndSize.first; - newInfo0.narSize = narHashAndSize.second; + { + PosixSourceAccessor accessor; + HashResult narHashAndSize = hashPath( + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256); + newInfo0.narHash = narHashAndSize.first; + newInfo0.narSize = narHashAndSize.second; + } assert(newInfo0.ca); return newInfo0; @@ -2531,7 +2512,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); + PosixSourceAccessor accessor; + HashResult narHashAndSize = hashPath( + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2741,7 +2725,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() .outPath = newInfo.path }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) - && drv->type().isPure()) + && !drv->type().isImpure()) { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 88152a645cf..f25cb942412 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -106,7 +106,7 @@ struct LocalDerivationGoal : public DerivationGoal RedirectedOutputs redirectedOutputs; /** - * The outputs paths used during the build. + * The output paths used during the build. * * - Input-addressed derivations or fixed content-addressed outputs are * sometimes built when some of their outputs already exist, and can not diff --git a/src/libstore/build/sandbox-defaults.sb b/src/libstore/build/sandbox-defaults.sb index 77f013aea7c..25ec1128555 100644 --- a/src/libstore/build/sandbox-defaults.sb +++ b/src/libstore/build/sandbox-defaults.sb @@ -68,6 +68,7 @@ R""( (allow file* (literal "/dev/null") (literal "/dev/random") + (literal "/dev/stderr") (literal "/dev/stdin") (literal "/dev/stdout") (literal "/dev/tty") diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 93867007d3a..c7e8e282508 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -2,6 +2,7 @@ #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" +#include "signals.hh" namespace nix { @@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun() thr = std::thread([this]() { try { + ReceiveInterrupts receiveInterrupts; + /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide.close(); }); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 9b8c36286dc..3a34f4006eb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -251,7 +251,7 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) void Worker::waitForBuildSlot(GoalPtr goal) { - debug("wait for build slot"); + goal->trace("wait for build slot"); bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) || (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) @@ -331,13 +331,23 @@ void Worker::run(const Goals & _topGoals) if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); } assert(!awake.empty()); @@ -449,7 +459,7 @@ void Worker::waitForInput() } else { printMsg(lvlVomit, "%1%: read %2% bytes", goal->getName(), rd); - std::string data((char *) buffer.data(), rd); + std::string_view data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); } @@ -519,7 +529,9 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.algo, store.printStorePath(path)); + HashResult current = hashPath( + *store.getFSAccessor(), CanonPath { store.printStorePath(path) }, + FileIngestionMethod::Recursive, info->narHash.algo); Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 23ad879148d..ced013ddd84 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -116,7 +116,7 @@ private: WeakGoals waitingForAWhile; /** - * Last time the goals in `waitingForAWhile` where woken up. + * Last time the goals in `waitingForAWhile` were woken up. */ steady_time_point lastWokenUp; diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index d201fb3ac5b..93558b49e23 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,13 @@ namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinUnpackChannel(const BasicDerivation & drv); +void builtinFetchurl( + const BasicDerivation & drv, + const std::map & outputs, + const std::string & netrcData); + +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 9283251ac0f..1ed7b39cc58 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -161,7 +161,9 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv(const BasicDerivation & drv) +void builtinBuildenv( + const BasicDerivation & drv, + const std::map & outputs) { auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); @@ -169,7 +171,7 @@ void builtinBuildenv(const BasicDerivation & drv) return i->second; }; - Path out = getAttr("out"); + auto out = outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 8bebd390d37..8e112e176e2 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -5,6 +5,9 @@ namespace nix { +/** + * Think of this as a "store level package attrset", but stripped down to no more than the needs of buildenv. + */ struct Package { Path path; bool active; @@ -42,6 +45,8 @@ typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv(const BasicDerivation & drv); +void builtinBuildenv( + const BasicDerivation & drv, + const std::map & outputs); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 2086bd0b919..b9dfeba2f8e 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,7 +6,10 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +void builtinFetchurl( + const BasicDerivation & drv, + const std::map & outputs, + const std::string & netrcData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -16,14 +19,15 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } - auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; + auto out = get(drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetchurl' requires an 'out' output"); + + if (!(drv.type().isFixed() || drv.type().isImpure())) + throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); - Path storePath = getAttr("out"); - auto mainUrl = getAttr("url"); + auto storePath = outputs.at("out"); + auto mainUrl = drv.env.at("url"); bool unpack = getOr(drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in @@ -59,13 +63,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) }; /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") + auto dof = std::get_if(&out->raw); + if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - std::optional ht = parseHashAlgoOpt(getAttr("outputHashAlgo")); - Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false)); + fetch(hashedMirror + printHashAlgo(dof->ca.hash.algo) + "/" + dof->ca.hash.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index ba04bb16cd8..6f68d4c0bc7 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,7 +3,9 @@ namespace nix { -void builtinUnpackChannel(const BasicDerivation & drv) +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs) { auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); @@ -11,7 +13,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) return i->second; }; - Path out = getAttr("out"); + auto out = outputs.at("out"); auto channelName = getAttr("channelName"); auto src = getAttr("src"); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index f42a131260a..2091f8e0267 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,7 +4,7 @@ namespace nix { -std::string makeFileIngestionPrefix(FileIngestionMethod m) +std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -16,10 +16,29 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) } } -std::string ContentAddressMethod::renderPrefix() const +std::string_view ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextIngestionMethod) -> std::string { return "text:"; }, + [](TextIngestionMethod) -> std::string_view { return "text"; }, + [](FileIngestionMethod m2) { + /* Not prefixed for back compat with things that couldn't produce text before. */ + return renderFileIngestionMethod(m2); + }, + }, raw); +} + +ContentAddressMethod ContentAddressMethod::parse(std::string_view m) +{ + if (m == "text") + return TextIngestionMethod {}; + else + return parseFileIngestionMethod(m); +} + +std::string_view ContentAddressMethod::renderPrefix() const +{ + return std::visit(overloaded { + [](TextIngestionMethod) -> std::string_view { return "text:"; }, [](FileIngestionMethod m2) { /* Not prefixed for back compat with things that couldn't produce text before. */ return makeFileIngestionPrefix(m2); @@ -38,7 +57,7 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return FileIngestionMethod::Flat; } -std::string ContentAddressMethod::render(HashAlgorithm ha) const +std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const { return std::visit(overloaded { [&](const TextIngestionMethod & th) { @@ -50,6 +69,18 @@ std::string ContentAddressMethod::render(HashAlgorithm ha) const }, raw); } +FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const +{ + return std::visit(overloaded { + [&](const TextIngestionMethod & th) { + return FileIngestionMethod::Flat; + }, + [&](const FileIngestionMethod & fim) { + return fim; + } + }, raw); +} + std::string ContentAddress::render() const { return std::visit(overloaded { @@ -79,7 +110,7 @@ static std::pair parseContentAddressMethodP prefix = *optPrefix; } - auto parseHashType_ = [&](){ + auto parseHashAlgorithm_ = [&](){ auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); @@ -90,7 +121,7 @@ static std::pair parseContentAddressMethodP // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashAlgorithm hashAlgo = parseHashType_(); + HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { TextIngestionMethod {}, std::move(hashAlgo), @@ -100,7 +131,7 @@ static std::pair parseContentAddressMethodP auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashAlgorithm hashAlgo = parseHashType_(); + HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { std::move(method), std::move(hashAlgo), @@ -121,7 +152,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) }; } -std::pair ContentAddressMethod::parse(std::string_view caMethod) +std::pair ContentAddressMethod::parseWithAlgo(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -143,7 +174,7 @@ std::string renderContentAddress(std::optional ca) std::string ContentAddress::printMethodAlgo() const { - return method.renderPrefix() + return std::string { method.renderPrefix() } + printHashAlgo(hash.algo); } @@ -176,13 +207,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con }, ca.method.raw); } -std::optional ContentAddressWithReferences::fromPartsOpt( - ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept +ContentAddressWithReferences ContentAddressWithReferences::fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs) { return std::visit(overloaded { - [&](TextIngestionMethod _) -> std::optional { + [&](TextIngestionMethod _) -> ContentAddressWithReferences { if (refs.self) - return std::nullopt; + throw Error("self-reference not allowed with text hashing"); return ContentAddressWithReferences { TextInfo { .hash = std::move(hash), @@ -190,7 +221,7 @@ std::optional ContentAddressWithReferences::fromPa } }; }, - [&](FileIngestionMethod m2) -> std::optional { + [&](FileIngestionMethod m2) -> ContentAddressWithReferences { return ContentAddressWithReferences { FixedOutputInfo { .method = m2, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 05234da382f..80538df5018 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include #include "hash.hh" #include "path.hh" +#include "file-content-address.hh" #include "comparator.hh" #include "variant-wrapper.hh" @@ -31,30 +32,14 @@ namespace nix { */ struct TextIngestionMethod : std::monostate { }; -/** - * An enumeration of the main ways we can serialize file system - * objects. - */ -enum struct FileIngestionMethod : uint8_t { - /** - * Flat-file hashing. Directly ingest the contents of a single file - */ - Flat = 0, - /** - * Recursive (or NAR) hashing. Serializes the file-system object in Nix - * Archive format and ingest that - */ - Recursive = 1 -}; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. */ -std::string makeFileIngestionPrefix(FileIngestionMethod m); +std::string_view makeFileIngestionPrefix(FileIngestionMethod m); /** - * An enumeration of all the ways we can serialize file system objects. + * An enumeration of all the ways we can content-address store objects. * * Just the type of a content address. Combine with the hash itself, and * we have a `ContentAddress` as defined below. Combine that, in turn, @@ -74,6 +59,20 @@ struct ContentAddressMethod MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); + /** + * Parse a content addressing method (name). + * + * The inverse of `render`. + */ + static ContentAddressMethod parse(std::string_view rawCaMethod); + + /** + * Render a content addressing method (name). + * + * The inverse of `parse`. + */ + std::string_view render() const; + /** * Parse the prefix tag which indicates how the files * were ingested, with the fixed output case not prefixed for back @@ -89,12 +88,12 @@ struct ContentAddressMethod * * The rough inverse of `parsePrefix()`. */ - std::string renderPrefix() const; + std::string_view renderPrefix() const; /** * Parse a content addressing method and hash type. */ - static std::pair parse(std::string_view rawCaMethod); + static std::pair parseWithAlgo(std::string_view rawCaMethod); /** * Render a content addressing method and hash type in a @@ -102,7 +101,15 @@ struct ContentAddressMethod * * The rough inverse of `parse()`. */ - std::string render(HashAlgorithm ha) const; + std::string renderWithAlgo(HashAlgorithm ht) const; + + /** + * Get the underlying way to content-address file system objects. + * + * Different ways of hashing store objects may use the same method + * for hashing file systeme objects. + */ + FileIngestionMethod getFileIngestionMethod() const; }; @@ -116,11 +123,11 @@ struct ContentAddressMethod * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the * following forms: * - * - ‘text:sha256:’: For paths - * computed by Store::makeTextPath() / Store::addTextToStore(). + * - `TextIngestionMethod`: + * ‘text:sha256:’ * - * - ‘fixed:::’: For paths computed by - * Store::makeFixedOutputPath() / Store::addToStore(). + * - `FixedIngestionMethod`: + * ‘fixed:::’ */ struct ContentAddress { @@ -266,11 +273,12 @@ struct ContentAddressWithReferences * * @param refs References to other store objects or oneself. * - * Do note that not all combinations are supported; `nullopt` is - * returns for invalid combinations. + * @note note that all combinations are supported. This is a + * *partial function* and exceptions will be thrown for invalid + * combinations. */ - static std::optional fromPartsOpt( - ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept; + static ContentAddressWithReferences fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs); ContentAddressMethod getMethod() const; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index a112d6d31d3..cf5020dfe08 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -119,7 +119,7 @@ struct TunnelLogger : public Logger if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { to << STDERR_ERROR << *ex; } else { - to << STDERR_ERROR << ex->what() << ex->status; + to << STDERR_ERROR << ex->what() << ex->info().status; } } } @@ -400,25 +400,12 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); + auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parseWithAlgo(camStr); auto hashAlgo = hashAlgo_; // work around clang bug FramedSource source(from); - // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. - return std::visit(overloaded { - [&](const TextIngestionMethod &) { - if (hashAlgo != HashAlgorithm::SHA256) - throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashAlgo(hashAlgo)); - // We could stream this by changing Store - std::string contents = source.drain(); - auto path = store->addTextToStore(name, contents, refs, repair); - return store->queryPathInfo(path); - }, - [&](const FileIngestionMethod & fim) { - auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs); - return store->queryPathInfo(path); - }, - }, contentAddressMethod.raw); + // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. + auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair); + return store->queryPathInfo(path); }(); logger->stopWork(); @@ -454,7 +441,7 @@ static void performOp(TunnelLogger * logger, ref store, eagerly consume the entire stream it's given, past the length of the Nar. */ TeeSource savedNARSource(from, saved); - NullParseSink sink; /* just parse the NAR */ + NullFileSystemObjectSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); } else { /* Incrementally parse the NAR file, stripping the @@ -496,7 +483,10 @@ static void performOp(TunnelLogger * logger, ref store, std::string s = readString(from); auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); - auto path = store->addTextToStore(suffix, s, refs, NoRepair); + auto path = ({ + StringSource source { s }; + store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + }); logger->stopWork(); to << store->printStorePath(path); break; @@ -923,7 +913,7 @@ static void performOp(TunnelLogger * logger, ref store, source = std::make_unique(from, to); else { TeeSource tee { from, saved }; - NullParseSink ether; + NullFileSystemObjectSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c35150b57b3..36042c06c39 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -2,6 +2,7 @@ #include "downstream-placeholder.hh" #include "store-api.hh" #include "globals.hh" +#include "types.hh" #include "util.hh" #include "split.hh" #include "common-protocol.hh" @@ -109,17 +110,17 @@ bool DerivationType::isSandboxed() const } -bool DerivationType::isPure() const +bool DerivationType::isImpure() const { return std::visit(overloaded { [](const InputAddressed & ia) { - return true; + return false; }, [](const ContentAddressed & ca) { - return true; + return false; }, [](const Impure &) { - return false; + return true; }, }, raw); } @@ -143,36 +144,89 @@ StorePath writeDerivation(Store & store, auto suffix = std::string(drv.name) + drvExtension; auto contents = drv.unparse(store, false); return readOnly || settings.readOnlyMode - ? store.computeStorePathForText(suffix, contents, references) - : store.addTextToStore(suffix, contents, references, repair); + ? store.makeFixedOutputPathFromCA(suffix, TextInfo { + .hash = hashString(HashAlgorithm::SHA256, contents), + .references = std::move(references), + }) + : ({ + StringSource s { contents }; + store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + }); +} + + +namespace { +/** + * This mimics std::istream to some extent. We use this much smaller implementation + * instead of plain istreams because the sentry object overhead is too high. + */ +struct StringViewStream { + std::string_view remaining; + + int peek() const { + return remaining.empty() ? EOF : remaining[0]; + } + + int get() { + if (remaining.empty()) return EOF; + char c = remaining[0]; + remaining.remove_prefix(1); + return c; + } +}; + +constexpr struct Escapes { + char map[256]; + constexpr Escapes() { + for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i; + map[(int) (unsigned char) 'n'] = '\n'; + map[(int) (unsigned char) 'r'] = '\r'; + map[(int) (unsigned char) 't'] = '\t'; + } + char operator[](char c) const { return map[(unsigned char) c]; } +} escapes; } /* Read string `s' from stream `str'. */ -static void expect(std::istream & str, std::string_view s) +static void expect(StringViewStream & str, std::string_view s) { - for (auto & c : s) { - if (str.get() != c) - throw FormatError("expected string '%1%'", s); - } + if (!str.remaining.starts_with(s)) + throw FormatError("expected string '%1%'", s); + str.remaining.remove_prefix(s.size()); } /* Read a C-style string from stream `str'. */ -static std::string parseString(std::istream & str) +static BackedStringView parseString(StringViewStream & str) { - std::string res; expect(str, "\""); - int c; - while ((c = str.get()) != '"') - if (c == '\\') { - c = str.get(); - if (c == 'n') res += '\n'; - else if (c == 'r') res += '\r'; - else if (c == 't') res += '\t'; - else res += c; + auto c = str.remaining.begin(), end = str.remaining.end(); + bool escaped = false; + for (; c != end && *c != '"'; c++) { + if (*c == '\\') { + c++; + if (c == end) + throw FormatError("unterminated string in derivation"); + escaped = true; + } + } + + const auto contentLen = c - str.remaining.begin(); + const auto content = str.remaining.substr(0, contentLen); + str.remaining.remove_prefix(contentLen + 1); + + if (!escaped) + return content; + + std::string res; + res.reserve(content.size()); + for (c = content.begin(), end = content.end(); c != end; c++) + if (*c == '\\') { + c++; + res += escapes[*c]; } - else res += c; + else res += *c; return res; } @@ -181,15 +235,15 @@ static void validatePath(std::string_view s) { throw FormatError("bad path '%1%' in derivation", s); } -static Path parsePath(std::istream & str) +static BackedStringView parsePath(StringViewStream & str) { auto s = parseString(str); - validatePath(s); + validatePath(*s); return s; } -static bool endOfList(std::istream & str) +static bool endOfList(StringViewStream & str) { if (str.peek() == ',') { str.get(); @@ -203,12 +257,12 @@ static bool endOfList(std::istream & str) } -static StringSet parseStrings(std::istream & str, bool arePaths) +static StringSet parseStrings(StringViewStream & str, bool arePaths) { StringSet res; expect(str, "["); while (!endOfList(str)) - res.insert(arePaths ? parsePath(str) : parseString(str)); + res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned()); return res; } @@ -261,7 +315,7 @@ static DerivationOutput parseDerivationOutput( } static DerivationOutput parseDerivationOutput( - const StoreDirConfig & store, std::istringstream & str, + const StoreDirConfig & store, StringViewStream & str, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); @@ -269,7 +323,7 @@ static DerivationOutput parseDerivationOutput( expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings); + return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings); } /** @@ -291,7 +345,7 @@ enum struct DerivationATermVersion { static DerivedPathMap::ChildNode parseDerivedPathMapNode( const StoreDirConfig & store, - std::istringstream & str, + StringViewStream & str, DerivationATermVersion version) { DerivedPathMap::ChildNode node; @@ -317,7 +371,7 @@ static DerivedPathMap::ChildNode parseDerivedPathMapNode( expect(str, ",["); while (!endOfList(str)) { expect(str, "("); - auto outputName = parseString(str); + auto outputName = parseString(str).toOwned(); expect(str, ","); node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); expect(str, ")"); @@ -343,7 +397,7 @@ Derivation parseDerivation( Derivation drv; drv.name = name; - std::istringstream str(std::move(s)); + StringViewStream str{s}; expect(str, "D"); DerivationATermVersion version; switch (str.peek()) { @@ -354,12 +408,12 @@ Derivation parseDerivation( case 'r': { expect(str, "rvWithVersion("); auto versionS = parseString(str); - if (versionS == "xp-dyn-drv") { + if (*versionS == "xp-dyn-drv") { // Only verison we have so far version = DerivationATermVersion::DynamicDerivations; xpSettings.require(Xp::DynamicDerivations); } else { - throw FormatError("Unknown derivation ATerm format version '%s'", versionS); + throw FormatError("Unknown derivation ATerm format version '%s'", *versionS); } expect(str, ","); break; @@ -371,7 +425,7 @@ Derivation parseDerivation( /* Parse the list of outputs. */ expect(str, "["); while (!endOfList(str)) { - expect(str, "("); std::string id = parseString(str); + expect(str, "("); std::string id = parseString(str).toOwned(); auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -380,28 +434,28 @@ Derivation parseDerivation( expect(str, ",["); while (!endOfList(str)) { expect(str, "("); - Path drvPath = parsePath(str); + auto drvPath = parsePath(str); expect(str, ","); - drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version)); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); - expect(str, ","); drv.platform = parseString(str); - expect(str, ","); drv.builder = parseString(str); + expect(str, ","); drv.platform = parseString(str).toOwned(); + expect(str, ","); drv.builder = parseString(str).toOwned(); /* Parse the builder arguments. */ expect(str, ",["); while (!endOfList(str)) - drv.args.push_back(parseString(str)); + drv.args.push_back(parseString(str).toOwned()); /* Parse the environment variables. */ expect(str, ",["); while (!endOfList(str)) { - expect(str, "("); auto name = parseString(str); - expect(str, ","); auto value = parseString(str); + expect(str, "("); auto name = parseString(str).toOwned(); + expect(str, ","); auto value = parseString(str).toOwned(); expect(str, ")"); - drv.env[name] = value; + drv.env.insert_or_assign(std::move(name), std::move(value)); } expect(str, ")"); @@ -547,7 +601,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)); + s += ','; printUnquotedString(s, std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -558,7 +612,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)); + s += ','; printUnquotedString(s, std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw); @@ -786,7 +840,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut }; } - if (!type.isPure()) { + if (type.isImpure()) { std::map outputHashes; for (const auto & [outputName, _] : drv.outputs) outputHashes.insert_or_assign(outputName, impureOutputHash); @@ -930,7 +984,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)) + << (std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -940,7 +994,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::Impure & doi) { out << "" - << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)) + << (std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo)) << "impure"; }, }, i.second.raw); @@ -1167,11 +1221,11 @@ nlohmann::json DerivationOutput::toJSON( // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo); + res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo); + res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo); res["impure"] = true; }, }, raw); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 2a326b57814..522523e4597 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -253,12 +253,17 @@ struct DerivationType { bool isSandboxed() const; /** - * Whether the derivation is expected to produce the same result - * every time, and therefore it only needs to be built once. This is - * only false for derivations that have the attribute '__impure = + * Whether the derivation is expected to produce a different result + * every time, and therefore it needs to be rebuilt every time. This is + * only true for derivations that have the attribute '__impure = * true'. + * + * Non-impure derivations can still behave impurely, to the degree permitted + * by the sandbox. Hence why this method isn't `isPure`: impure derivations + * are not the negation of pure derivations. Purity can not be ascertained + * except by rather heavy tools. */ - bool isPure() const; + bool isImpure() const; /** * Does the derivation knows its own output paths? diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 3105dbc931f..a7b40432108 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -12,9 +12,9 @@ namespace nix { bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ { \ const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields1 = std::tie(*me->drvPath, me->FIELD); \ me = &other; \ - auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields2 = std::tie(*me->drvPath, me->FIELD); \ return fields1 COMPARATOR fields2; \ } #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ @@ -22,13 +22,9 @@ namespace nix { CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) -#define FIELD_TYPE std::string CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) -#undef FIELD_TYPE -#define FIELD_TYPE OutputsSpec CMP(SingleDerivedPath, DerivedPathBuilt, outputs) -#undef FIELD_TYPE #undef CMP #undef CMP_ONE diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 821cda399c9..e4f13b8f410 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -58,12 +58,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair, CheckSigsFlag checkSigs) override { unsupported("addToStore"); } - StorePath addTextToStore( + virtual StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override + { unsupported("addToStore"); } void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index d57b25bd70c..cb36c0c1b47 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - NullParseSink ether; + NullFileSystemObjectSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 9ddcfedd8a6..bab21bf517b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -895,12 +895,12 @@ template FileTransferError::FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args) : Error(args...), error(error), response(response) { - const auto hf = hintfmt(args...); + const auto hf = HintFmt(args...); // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. if (response && (response->size() < 1024 || response->find("") != std::string::npos)) - err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); + err.msg = HintFmt("%1%\n\nresponse body:\n\n%2%", Uncolored(hf.str()), chomp(*response)); else err.msg = hf; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 2bd3a2edca0..cb820e2d500 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -149,11 +149,12 @@ void LocalStore::addTempRoot(const StorePath & path) try { nix::connect(fdRootsSocket->get(), socketPath); } catch (SysError & e) { - /* The garbage collector may have exited, so we need to - restart. */ - if (e.errNo == ECONNREFUSED) { - debug("GC socket connection refused"); + /* The garbage collector may have exited or not + created the socket yet, so we need to restart. */ + if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) { + debug("GC socket connection refused: %s", e.msg()); fdRootsSocket->close(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); goto restart; } throw; @@ -413,7 +414,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); - } catch (SysError & e) { + } catch (SystemError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; throw; @@ -509,6 +510,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock..."); + /* Synchronisation point to test ENOENT handling in + addTempRoot(), see tests/gc-non-blocking.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC_1")) + readFile(*p); + /* Start the server for receiving new roots. */ auto socketPath = stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); @@ -632,6 +638,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) roots.insert(root.first); } + /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC_2")) + readFile(*p); + /* Helper function that deletes a path from the store and throws GCLimitReached if we've deleted enough garbage. */ auto deleteFromStore = [&](std::string_view baseName) @@ -778,10 +788,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */ - if (auto p = getEnv("_NIX_TEST_GC_SYNC")) - readFile(*p); - /* Either delete all garbage paths, or just the specified paths (for gcDeleteSpecific). */ if (options.action == GCOptions::gcDeleteSpecific) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f401d076dde..d22ae4ca02e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -15,8 +15,6 @@ #include -#include - #ifdef __GLIBC__ # include # include @@ -120,7 +118,7 @@ void loadConfFile() try { std::string contents = readFile(path); globalConfig.applyConfig(contents, path); - } catch (SysError &) { } + } catch (SystemError &) { } }; applyConfigFile(settings.nixConfDir + "/nix.conf"); @@ -409,9 +407,6 @@ void initLibStore() { initLibUtil(); - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - loadConfFile(); preloadNSS(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index df977e294ff..8330d6571d5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -144,20 +144,25 @@ public: */ bool verboseBuild = true; - Setting logLines{this, 10, "log-lines", + Setting logLines{this, 25, "log-lines", "The number of lines of the tail of " "the log to show if a build fails."}; MaxBuildJobsSetting maxBuildJobs{ this, 1, "max-jobs", R"( - This option defines the maximum number of jobs that Nix will try to - build in parallel. The default is `1`. The special value `auto` - causes Nix to use the number of CPUs in your system. `0` is useful - when using remote builders to prevent any local builds (except for - `preferLocalBuild` derivation attribute which executes locally - regardless). It can be overridden using the `--max-jobs` (`-j`) - command line switch. + Maximum number of jobs that Nix will try to build locally in parallel. + + The special value `auto` causes Nix to use the number of CPUs in your system. + Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders). + This will not affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally. + + > **Note** + > + > The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting. + + + The setting can be overridden using the `--max-jobs` (`-j`) command line switch. )", {"build-max-jobs"}}; @@ -175,14 +180,21 @@ public: getDefaultCores(), "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It can be overridden using the `--cores` - command line switch and defaults to `1`. The value `0` means that - the builder should use all available CPU cores in the system. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation. + The `builder` executable can use this variable to control its own maximum amount of parallelism. + + + For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it will pass the `-j${NIX_BUILD_CORES}` flag to GNU Make. + + The value `0` means that the `builder` should use all available CPU cores in the system. + + > **Note** + > + > The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting. )", {"build-cores"}, // Don't document the machine-specific default value @@ -214,7 +226,11 @@ public: In general, you do not have to modify this setting. While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. - This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). + This value is available in the Nix language as + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + if the + [`eval-system`](#conf-eval-system) + configuration option is set as the empty string. )"}; Setting maxSilentTime{ @@ -261,9 +277,121 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", R"( - A semicolon-separated list of build machines. - For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) - )"}; + A semicolon- or newline-separated list of build machines. + + In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`. + + > **Example** + > + > This is the default setting: + > + > ``` + > builders = @/etc/nix/machines + > ``` + + Each machine specification consists of the following elements, separated by spaces. + Only the first element is required. + To leave a field at its default, set it to `-`. + + 1. The URI of the remote store in the format `ssh://[username@]hostname`. + + > **Example** + > + > `ssh://nix@mac` + + For backward compatibility, `ssh://` may be omitted. + The hostname may be an alias defined in `~/.ssh/config`. + + 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + If omitted, this defaults to the local platform type. + + > **Example** + > + > `aarch64-darwin` + + It is possible for a machine to support multiple platform types. + + > **Example** + > + > `i686-linux,x86_64-linux` + + 3. The SSH identity file to be used to log in to the remote machine. + If omitted, SSH will use its regular identities. + + > **Example** + > + > `/home/user/.ssh/id_mac` + + 4. The maximum number of builds that Nix will execute in parallel on the machine. + Typically this should be equal to the number of CPU cores. + + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + If there are multiple machines of the right type, Nix will prefer the fastest, taking load into account. + + 6. A comma-separated list of supported [system features](#conf-system-features). + + A machine will only be used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine. + + 7. A comma-separated list of required [system features](#conf-system-features). + + A machine will only be used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute. + + 8. The (base64-encoded) public host key of the remote machine. + If omitted, SSH will use its regular `known_hosts` file. + + The value for this field can be obtained via `base64 -w0`. + + > **Example** + > + > Multiple builders specified on the command line: + > + > ```console + > --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + > ``` + + > **Example** + > + > This specifies several machines that can perform `i686-linux` builds: + > + > ``` + > nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm + > nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2 + > nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark + > ``` + > + > However, `poochie` will only build derivations that have the attribute + > + > ```nix + > requiredSystemFeatures = [ "benchmark" ]; + > ``` + > + > or + > + > ```nix + > requiredSystemFeatures = [ "benchmark" "kvm" ]; + > ``` + > + > `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds. + > For regular builds, `itchy` will be preferred over `scratchy` because it has a higher speed factor. + + For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list. + + > **Note** + > + > A build machine must be accessible via SSH and have Nix installed. + > `nix` must be available in `$PATH` for the user connecting over SSH. + + > **Warning** + > + > If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`). + > + > If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine. + + To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0. + + If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substituters) to `true`. + )", + {}, false}; Setting alwaysAllowSubstitutes{ this, false, "always-allow-substitutes", @@ -627,11 +755,11 @@ public: At least one of the following condition must be met for Nix to accept copying a store object from another - Nix store (such as a substituter): + Nix store (such as a [substituter](#conf-substituters)): - the store object has been signed using a key in the trusted keys list - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) + - the store object is [content-addressed](@docroot@/glossary.md#gloss-content-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -784,10 +912,17 @@ public: Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( - The TTL in seconds for negative lookups. If a store path is queried - from a substituter but was not found, there will be a negative - lookup cached in the local disk cache database for the specified - duration. + The TTL in seconds for negative lookups. + If a store path is queried from a [substituter](#conf-substituters) but was not found, there will be a negative lookup cached in the local disk cache database for the specified duration. + + Set to `0` to force updating the lookup cache. + + To wipe the lookup cache completely: + + ```shell-session + $ rm $HOME/.cache/nix/binary-cache-v*.sqlite* + # rm /root/.cache/nix/binary-cache-v*.sqlite* + ``` )"}; Setting ttlPositiveNarInfoCache{ @@ -942,7 +1077,9 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; +#endif +#if HAVE_ACL_SUPPORT Setting ignoredAcls{ this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls", R"( diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc new file mode 100644 index 00000000000..70478e7ad01 --- /dev/null +++ b/src/libstore/keys.cc @@ -0,0 +1,31 @@ +#include "file-system.hh" +#include "globals.hh" +#include "keys.hh" + +namespace nix { + +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.trustedPublicKeys.get()) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.secretKeyFiles.get()) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SystemError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + +} diff --git a/src/libstore/keys.hh b/src/libstore/keys.hh new file mode 100644 index 00000000000..3da19493fbb --- /dev/null +++ b/src/libstore/keys.hh @@ -0,0 +1,10 @@ +#pragma once +///@file + +#include "signature/local-keys.hh" + +namespace nix { + +PublicKeys getDefaultPublicKeys(); + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 06bef9d08cf..e422adeec85 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -22,45 +22,10 @@ std::string LegacySSHStoreConfig::doc() } -struct LegacySSHStore::Connection +struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection { std::unique_ptr sshConn; - FdSink to; - FdSource from; - ServeProto::Version remoteVersion; bool good = true; - - /** - * Coercion to `ServeProto::ReadConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::ReadConn () - { - return ServeProto::ReadConn { - .from = from, - .version = remoteVersion, - }; - } - - /* - * Coercion to `ServeProto::WriteConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::WriteConn () - { - return ServeProto::WriteConn { - .to = to, - .version = remoteVersion, - }; - } }; @@ -90,34 +55,31 @@ LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & h ref LegacySSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --serve --write", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + Strings command = remoteProgram.get(); + command.push_back("--serve"); + command.push_back("--write"); + if (remoteStore.get() != "") { + command.push_back("--store"); + command.push_back(remoteStore.get()); + } + conn->sshConn = master.startCommand(std::move(command)); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); + StringSink saved; + TeeSource tee(conn->from, saved); try { - conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; - conn->to.flush(); - - StringSink saved; - try { - TeeSource tee(conn->from, saved); - unsigned int magic = readInt(tee); - if (magic != SERVE_MAGIC_2) - throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - } catch (SerialisationError & e) { - /* In case the other side is waiting for our input, - close it. */ - conn->sshConn->in.close(); - auto msg = conn->from.drain(); - throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s + msg)); + conn->remoteVersion = ServeProto::BasicClientConnection::handshake( + conn->to, tee, SERVE_PROTOCOL_VERSION, host); + } catch (SerialisationError & e) { + // in.close(): Don't let the remote block on us not writing. + conn->sshConn->in.close(); + { + NullSink nullSink; + conn->from.drainInto(nullSink); } - conn->remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - + throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", + host, chomp(saved.s)); } catch (EndOfFile & e) { throw Error("cannot connect to '%1%'", host); } @@ -232,16 +194,16 @@ void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) } -void LegacySSHStore::putBuildSettings(Connection & conn) +static ServeProto::BuildOptions buildSettings() { - ServeProto::write(*this, conn, ServeProto::BuildOptions { + return { .maxSilentTime = settings.maxSilentTime, .buildTimeout = settings.buildTimeout, .maxLogSize = settings.maxLogSize, .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway .enforceDeterminism = 0, .keepFailed = settings.keepFailed, - }); + }; } @@ -250,14 +212,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas { auto conn(connections->get()); - conn->to - << ServeProto::Command::BuildDerivation - << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - - putBuildSettings(*conn); - - conn->to.flush(); + conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings()); return ServeProto::Serialise::read(*this, *conn); } @@ -288,7 +243,7 @@ void LegacySSHStore::buildPaths(const std::vector & drvPaths, Build } conn->to << ss; - putBuildSettings(*conn); + ServeProto::write(*this, *conn, buildSettings()); conn->to.flush(); @@ -328,15 +283,8 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryValidPaths - << false // lock - << maybeSubstitute; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); + return conn->queryValidPaths(*this, + false, paths, maybeSubstitute); } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index c40c256bbea..ae890177b3d 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -13,7 +13,7 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{this, "nix-store", "remote-program", + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; const Setting maxConnections{this, 1, "max-connections", @@ -59,25 +59,24 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { unsupported("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override - { unsupported("addToStore"); } - - StorePath addTextToStore( std::string_view name, - std::string_view s, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, const StorePathSet & references, + PathFilter & filter, RepairFlag repair) override - { unsupported("addTextToStore"); } - -private: + { unsupported("addToStore"); } - void putBuildSettings(Connection & conn); + virtual StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override + { unsupported("addToStore"); } public: diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 953f3a264eb..81c385ddb9e 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -28,7 +28,7 @@ struct LocalStoreAccessor : PosixSourceAccessor auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); + return CanonPath(store->getRealStoreDir()) / storePath.to_string() / CanonPath(rest); } std::optional maybeLstat(const CanonPath & path) override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7e82bae282e..2c22bfe319d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -13,11 +13,15 @@ #include "compression.hh" #include "signals.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" +#include "keys.hh" #include #include #include +#include +#include #include #include #include @@ -272,7 +276,7 @@ LocalStore::LocalStore(const Params & params) [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); } } - } catch (SysError & e) { /* don't care about errors */ + } catch (SystemError & e) { /* don't care about errors */ } /* Acquire the big fat lock in shared mode to make sure that no @@ -1044,8 +1048,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - NullParseSink sink; - parseDump(sink, source); + NullFileSystemObjectSink sink; + try { + parseDump(sink, source); + } catch (...) { + ignoreException(); + } } }; @@ -1088,11 +1096,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca) { auto & specified = *info.ca; - auto actualHash = hashCAPath( - specified.method, - specified.hash.algo, - info.path - ); + auto actualHash = ({ + HashModuloSink caSink { + specified.hash.algo, + std::string { info.path.hashPart() }, + }; + PosixSourceAccessor accessor; + dumpPath( + *getFSAccessor(false), + CanonPath { printStorePath(info.path) }, + caSink, + specified.method.getFileIngestionMethod()); + ContentAddress { + .method = specified.method, + .hash = caSink.finish().first, + }; + }); if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), @@ -1115,8 +1134,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath LocalStore::addToStoreFromDump( + Source & source0, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1130,7 +1154,11 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name path. */ bool inMemory = false; - std::string dump; + struct Free { + void operator()(void* v) { free(v); } + }; + std::unique_ptr dumpBuffer(nullptr); + std::string_view dump; /* Fill out buffer, and decide whether we are working strictly in memory based on whether we break out because the buffer is full @@ -1139,13 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto oldSize = dump.size(); constexpr size_t chunkSize = 65536; auto want = std::min(chunkSize, settings.narBufferSize - oldSize); - dump.resize(oldSize + want); + if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) { + dumpBuffer.release(); + dumpBuffer.reset((char*) tmp); + } else { + throw std::bad_alloc(); + } auto got = 0; Finally cleanup([&]() { - dump.resize(oldSize + got); + dump = {dumpBuffer.get(), dump.size() + got}; }); try { - got = source.read(dump.data() + oldSize, want); + got = source.read(dumpBuffer.get() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; @@ -1166,25 +1199,22 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource); - else - writeFile(tempPath, bothSource); + restorePath(tempPath, bothSource, method.getFileIngestionMethod()); - dump.clear(); + dumpBuffer.reset(); + dump = {}; } auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - .method = method, - .hash = hash, - .references = { + auto desc = ContentAddressWithReferences::fromParts( + method, + hash, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }; + }); auto dstPath = makeFixedOutputPathFromCA(name, desc); @@ -1207,11 +1237,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name if (inMemory) { StringSource dumpSource { dump }; - /* Restore from the NAR in memory. */ - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource); - else - writeFile(realPath, dumpSource); + /* Restore from the buffer in memory. */ + restorePath(realPath, dumpSource, method.getFileIngestionMethod()); } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); @@ -1247,58 +1274,6 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name } -StorePath LocalStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, RepairFlag repair) -{ - auto hash = hashString(HashAlgorithm::SHA256, s); - auto dstPath = makeTextPath(name, TextInfo { - .hash = hash, - .references = references, - }); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - writeFile(realPath, s); - - canonicalisePathMetaData(realPath, {}); - - StringSink sink; - dumpString(s, sink); - auto narHash = hashString(HashAlgorithm::SHA256, sink.s); - - optimisePath(realPath, repair); - - ValidPathInfo info { dstPath, narHash }; - info.narSize = sink.s.size(); - info.references = references; - info.ca = { - .method = TextIngestionMethod {}, - .hash = hash, - }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - /* Create a temporary directory in the store that won't be garbage-collected until the returned FD is closed. */ std::pair LocalStore::createTempDirInStore() @@ -1389,7 +1364,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false); + PosixSourceAccessor accessor; + std::string hash = hashPath( + accessor, CanonPath { linkPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1605,7 +1583,8 @@ void LocalStore::signRealisation(Realisation & realisation) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - realisation.sign(secretKey); + LocalSigner signer(std::move(secretKey)); + realisation.sign(signer); } } @@ -1617,7 +1596,8 @@ void LocalStore::signPathInfo(ValidPathInfo & info) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - info.sign(*this, secretKey); + LocalSigner signer(std::move(secretKey)); + info.sign(*this, signer); } } @@ -1696,42 +1676,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, const HashAlgorithm & hashAlgo, - const StorePath & path) -{ - return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart()); -} - -ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, - const HashAlgorithm & hashAlgo, - const Path & path, - const std::string_view pathHash -) -{ - HashModuloSink caSink ( hashAlgo, std::string(pathHash) ); - std::visit(overloaded { - [&](const TextIngestionMethod &) { - readFile(path, caSink); - }, - [&](const FileIngestionMethod & m2) { - switch (m2) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - }, - }, method.raw); - return ContentAddress { - .method = method, - .hash = caSink.finish().first, - }; -} - void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) { assert(drvPath.isDerivation()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ee605b5a2df..ba56d3ead12 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -177,12 +177,11 @@ public: void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; - - StorePath addTextToStore( + StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, + ContentAddressMethod method, + HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) override; @@ -350,19 +349,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - // XXX: Make a generic `Store` method - ContentAddress hashCAPath( - const ContentAddressMethod & method, - const HashAlgorithm & hashAlgo, - const StorePath & path); - - ContentAddress hashCAPath( - const ContentAddressMethod & method, - const HashAlgorithm & hashAlgo, - const Path & path, - const std::string_view pathHash - ); - void addBuildLog(const StorePath & drvPath, std::string_view log) override; friend struct LocalDerivationGoal; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 68ccdc409be..f86643849dd 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil -libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS) ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif @@ -16,15 +16,15 @@ endif $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp endif ifdef HOST_SOLARIS - libstore_LDFLAGS += -lsocket + libstore_LDFLAGS += -lsocket endif ifeq ($(HAVE_SECCOMP), 1) - libstore_LDFLAGS += $(LIBSECCOMP_LIBS) + libstore_LDFLAGS += $(LIBSECCOMP_LIBS) endif libstore_CXXFLAGS += \ @@ -48,9 +48,9 @@ $(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp @mv $@.tmp $@ else -ifneq ($(sandbox_shell),) -libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" -endif + ifneq ($(sandbox_shell),) + libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" + endif endif $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 512115893ed..2d461c63afb 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -32,11 +32,19 @@ Machine::Machine(decltype(storeUri) storeUri, systemTypes(systemTypes), sshKey(sshKey), maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), + speedFactor(speedFactor == 0.0f ? 1.0f : std::move(speedFactor)), supportedFeatures(supportedFeatures), mandatoryFeatures(mandatoryFeatures), sshPublicHostKey(sshPublicHostKey) -{} +{ + if (speedFactor < 0.0) + throw UsageError("speed factor must be >= 0"); +} + +bool Machine::systemSupported(const std::string & system) const +{ + return system == "builtin" || (systemTypes.count(system) > 0); +} bool Machine::allSupported(const std::set & features) const { @@ -130,6 +138,14 @@ static Machine parseBuilderLine(const std::string & line) return result.value(); }; + auto parseFloatField = [&](size_t fieldIndex) { + const auto result = string2Int(tokens[fieldIndex]); + if (!result) { + throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line); + } + return result.value(); + }; + auto ensureBase64 = [&](size_t fieldIndex) { const auto & str = tokens[fieldIndex]; try { @@ -145,10 +161,10 @@ static Machine parseBuilderLine(const std::string & line) return { tokens[0], - isSet(1) ? tokenizeString>(tokens[1], ",") : std::vector{settings.thisSystem}, + isSet(1) ? tokenizeString>(tokens[1], ",") : std::set{settings.thisSystem}, isSet(2) ? tokens[2] : "", isSet(3) ? parseUnsignedIntField(3) : 1U, - isSet(4) ? parseUnsignedIntField(4) : 1U, + isSet(4) ? parseFloatField(4) : 1.0f, isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, isSet(7) ? ensureBase64(7) : "" diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 1adeaf1f07f..8516409d48a 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -10,17 +10,30 @@ class Store; struct Machine { const std::string storeUri; - const std::vector systemTypes; + const std::set systemTypes; const std::string sshKey; const unsigned int maxJobs; - const unsigned int speedFactor; + const float speedFactor; const std::set supportedFeatures; const std::set mandatoryFeatures; const std::string sshPublicHostKey; bool enabled = true; + /** + * @return Whether `system` is either `"builtin"` or in + * `systemTypes`. + */ + bool systemSupported(const std::string & system) const; + + /** + * @return Whether `features` is a subset of the union of `supportedFeatures` and + * `mandatoryFeatures` + */ bool allSupported(const std::set & features) const; + /** + * @return @Whether `mandatoryFeatures` is a subset of `features` + */ bool mandatoryMet(const std::set & features) const; Machine(decltype(storeUri) storeUri, diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 15b05fe25fe..cecf8148f45 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -19,6 +19,35 @@ struct NarMember std::map children; }; +struct NarMemberConstructor : CreateRegularFileSink +{ +private: + + NarMember & narMember; + + uint64_t & pos; + +public: + + NarMemberConstructor(NarMember & nm, uint64_t & pos) + : narMember(nm), pos(pos) + { } + + void isExecutable() override + { + narMember.stat.isExecutable = true; + } + + void preallocateContents(uint64_t size) override + { + narMember.stat.fileSize = size; + narMember.stat.narOffset = pos; + } + + void operator () (std::string_view data) override + { } +}; + struct NarAccessor : public SourceAccessor { std::optional nar; @@ -27,7 +56,7 @@ struct NarAccessor : public SourceAccessor NarMember root; - struct NarIndexer : ParseSink, Source + struct NarIndexer : FileSystemObjectSink, Source { NarAccessor & acc; Source & source; @@ -42,7 +71,7 @@ struct NarAccessor : public SourceAccessor : acc(acc), source(source) { } - void createMember(const Path & path, NarMember member) + NarMember & createMember(const Path & path, NarMember member) { size_t level = std::count(path.begin(), path.end(), '/'); while (parents.size() > level) parents.pop(); @@ -50,11 +79,14 @@ struct NarAccessor : public SourceAccessor if (parents.empty()) { acc.root = std::move(member); parents.push(&acc.root); + return acc.root; } else { if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); - parents.push(&result.first->second); + auto & ref = result.first->second; + parents.push(&ref); + return ref; } } @@ -68,34 +100,18 @@ struct NarAccessor : public SourceAccessor } }); } - void createRegularFile(const Path & path) override + void createRegularFile(const Path & path, std::function func) override { - createMember(path, NarMember{ .stat = { + auto & nm = createMember(path, NarMember{ .stat = { .type = Type::tRegular, .fileSize = 0, .isExecutable = false, .narOffset = 0 } }); + NarMemberConstructor nmc { nm, pos }; + func(nmc); } - void closeRegularFile() override - { } - - void isExecutable() override - { - parents.top()->stat.isExecutable = true; - } - - void preallocateContents(uint64_t size) override - { - auto & st = parents.top()->stat; - st.fileSize = size; - st.narOffset = pos; - } - - void receiveContents(std::string_view data) override - { } - void createSymlink(const Path & path, const std::string & target) override { createMember(path, @@ -261,7 +277,7 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) json &res2 = obj["entries"]; for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + name, true); + res2[name] = listNar(accessor, path / name, true); } else res2[name] = json::object(); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 310105c758a..07beb8acb0b 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -209,7 +209,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - assert(r.next()); + if (!r.next()) { abort(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b395453d157..78e4f6d86b2 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "signals.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -146,7 +147,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(HashAlgorithm::SHA256, path).first; + Hash hash = ({ + PosixSourceAccessor accessor; + hashPath( + accessor, CanonPath { path }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first; + }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ @@ -156,7 +162,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first)) + || (repair && hash != ({ + PosixSourceAccessor accessor; + hashPath( + accessor, CanonPath { linkPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first; + }))) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); @@ -231,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ try { renameFile(tempLink, path); - } catch (SysError & e) { + } catch (SystemError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index f58e31bfd5f..d82ccd0c951 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -38,9 +38,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const } -void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) +void ValidPathInfo::sign(const Store & store, const Signer & signer) { - sigs.insert(secretKey.signDetached(fingerprint(store))); + sigs.insert(signer.signDetached(fingerprint(store))); } std::optional ValidPathInfo::contentAddressWithReferences() const diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 077abc7e15b..b6dc0855d75 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "crypto.hh" +#include "signature/signer.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -107,7 +107,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo { */ std::string fingerprint(const Store & store) const; - void sign(const Store & store, const SecretKey & secretKey); + void sign(const Store & store, const Signer & signer); /** * @return The `ContentAddressWithReferences` that determines the diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index a44e6a2eb52..56c2cfc1db5 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -3,6 +3,11 @@ namespace nix { -static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)"; + +static constexpr std::string_view nameRegexStr = + // This uses a negative lookahead: (?!\.\.?(-|$)) + // - deny ".", "..", or those strings followed by '-' + // - when it's not those, start again at the start of the input and apply the next regex, which is [0-9a-zA-Z\+\-\._\?=]+ + R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)"; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 1afd10af78a..5db4b974c3c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,7 +1,5 @@ #include "store-dir-config.hh" -#include - namespace nix { static void checkName(std::string_view path, std::string_view name) @@ -11,9 +9,20 @@ static void checkName(std::string_view path, std::string_view name) if (name.size() > StorePath::MaxPathLen) throw BadStorePath("store path '%s' has a name longer than %d characters", path, StorePath::MaxPathLen); - if (name[0] == '.') - throw BadStorePath("store path '%s' starts with illegal character '.'", path); // See nameRegexStr for the definition + if (name[0] == '.') { + // check against "." and "..", followed by end or dash + if (name.size() == 1) + throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + if (name[1] == '-') + throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "."); + if (name[1] == '.') { + if (name.size() == 2) + throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + if (name[2] == '-') + throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".."); + } + } for (auto c : name) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') @@ -49,9 +58,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(HashAlgorithm::SHA1); - randombytes_buf(hash.hash, hash.hashSize); - return StorePath(hash, name); + return StorePath(Hash::random(HashAlgorithm::SHA1), name); } StorePath StoreDirConfig::parseStorePath(std::string_view path) const diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index f38fa83690b..8b29e90d48f 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -1,4 +1,4 @@ -#if HAVE_SYS_XATTR_H +#if HAVE_ACL_SUPPORT # include #endif @@ -78,7 +78,7 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#ifdef HAVE_SYS_XATTR_H +#if HAVE_ACL_SUPPORT /* Remove extended attributes / ACLs. */ ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 93ddb5b20df..86bfdd1a8bf 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,7 @@ #include "realisation.hh" #include "store-api.hh" #include "closure.hh" +#include "signature/local-keys.hh" #include namespace nix { @@ -113,9 +114,9 @@ std::string Realisation::fingerprint() const return serialized.dump(); } -void Realisation::sign(const SecretKey & secretKey) +void Realisation::sign(const Signer &signer) { - signatures.insert(secretKey.signDetached(fingerprint())); + signatures.insert(signer.signDetached(fingerprint())); } bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 4ba2123d81c..ddb4af770a2 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -8,7 +8,7 @@ #include "derived-path.hh" #include #include "comparator.hh" -#include "crypto.hh" +#include "signature/signer.hh" namespace nix { @@ -64,7 +64,7 @@ struct Realisation { static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); std::string fingerprint() const; - void sign(const SecretKey &); + void sign(const Signer &); bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 03e57a565d0..b44edfe890d 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -87,13 +87,13 @@ std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPat nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; - } catch (SysError &) { } + } catch (SystemError &) { } try { auto narAccessor = makeNarAccessor(nix::readFile(cacheFile)); nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; - } catch (SysError &) { } + } catch (SystemError &) { } } StringSink sink; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cc26c2a9436..fadef45ffcc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -16,6 +16,8 @@ #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include "signals.hh" + #include namespace nix { @@ -65,6 +67,7 @@ void RemoteStore::initConnection(Connection & conn) { /* Send the magic greeting, check for the reply. */ try { + conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)"; conn.to << WORKER_MAGIC_1; conn.to.flush(); StringSink saved; @@ -186,7 +189,7 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos && m.find("Derive([") != std::string::npos) - throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); + throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); } throw; } @@ -225,7 +228,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute conn->to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(*this, *conn, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { - conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + conn->to << maybeSubstitute; } conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); @@ -432,7 +435,7 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << caMethod.render(hashAlgo); + << caMethod.renderWithAlgo(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; @@ -502,8 +505,13 @@ ref RemoteStore::addCAToStore( } -StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath RemoteStore::addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; } @@ -603,16 +611,6 @@ void RemoteStore::addMultipleToStore( } -StorePath RemoteStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - StringSource source(s); - return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path; -} - void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); @@ -1071,6 +1069,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function::read(store, *this); +} + + +void ServeProto::BasicClientConnection::putBuildDerivationRequest( + const Store & store, + const StorePath & drvPath, const BasicDerivation & drv, + const ServeProto::BuildOptions & options) +{ + to + << ServeProto::Command::BuildDerivation + << store.printStorePath(drvPath); + writeDerivation(to, store, drv); + + ServeProto::write(store, *this, options); + + to.flush(); +} + +} diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index 6f3b177acd2..fd8d946970e 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -10,6 +10,7 @@ #include "serve-protocol.hh" #include "length-prefixed-protocol-helper.hh" +#include "store-api.hh" namespace nix { @@ -56,4 +57,101 @@ struct ServeProto::Serialise /* protocol-specific templates */ +struct ServeProto::BasicClientConnection +{ + FdSink to; + FdSource from; + ServeProto::Version remoteVersion; + + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param host Just used to add context to thrown exceptions. + */ + static ServeProto::Version handshake( + BufferedSink & to, + Source & from, + ServeProto::Version localVersion, + std::string_view host); + + /** + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator ServeProto::ReadConn () + { + return ServeProto::ReadConn { + .from = from, + .version = remoteVersion, + }; + } + + /** + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator ServeProto::WriteConn () + { + return ServeProto::WriteConn { + .to = to, + .version = remoteVersion, + }; + } + + StorePathSet queryValidPaths( + const Store & remoteStore, + bool lock, const StorePathSet & paths, + SubstituteFlag maybeSubstitute); + + /** + * Just the request half, because Hydra may do other things between + * issuing the request and reading the `BuildResult` response. + */ + void putBuildDerivationRequest( + const Store & store, + const StorePath & drvPath, const BasicDerivation & drv, + const ServeProto::BuildOptions & options); +}; + +struct ServeProto::BasicServerConnection +{ + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + */ + static ServeProto::Version handshake( + BufferedSink & to, + Source & from, + ServeProto::Version localVersion); +}; + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 1665b935fd6..8c112bb74c7 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -59,6 +59,14 @@ struct ServeProto Version version; }; + /** + * Stripped down serialization logic suitable for sharing with Hydra. + * + * @todo remove once Hydra uses Store abstraction consistently. + */ + struct BasicClientConnection; + struct BasicServerConnection; + /** * Data type for canonical pairs of serialisers for the serve protocol. * diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index d7432a3059e..06abfb90bb5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -10,19 +10,19 @@ namespace nix { -SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf) +SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf) : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset) { auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": "; - err.msg = hintfmt("%s: %s%s, %s (in '%s')", - normaltxt(hf.str()), + err.msg = HintFmt("%s: %s%s, %s (in '%s')", + Uncolored(hf.str()), offsetStr, sqlite3_errstr(extendedErrNo), errMsg, path ? path : "(in-memory)"); } -[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf) +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, HintFmt && hf) { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); @@ -33,7 +33,7 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf)); - exp.err.msg = hintfmt( + exp.err.msg = HintFmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" : "SQLite database '%s' is busy", @@ -249,7 +249,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) if (now > nextWarning) { nextWarning = now + 10; logWarning({ - .msg = hintfmt(e.what()) + .msg = HintFmt(e.what()) }); } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 0c08267f70b..003e4d1013d 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -142,19 +142,19 @@ struct SQLiteError : Error template [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { - throw_(db, hintfmt(fs, args...)); + throw_(db, HintFmt(fs, args...)); } - SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf); + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf); protected: template SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args) - : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...)) + : SQLiteError(path, errMsg, errNo, extendedErrNo, offset, HintFmt(fs, args...)) { } - [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); + [[noreturn]] static void throw_(sqlite3 * db, HintFmt && hf); }; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index d4c8ab5b2f0..0cf92b114c7 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -17,7 +17,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{this, "nix-daemon", "remote-program", + const Setting remoteProgram{this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; const std::string name() override { return "Experimental SSH Store"; } @@ -212,14 +212,15 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH ref SSHStore::openConnection() { auto conn = make_ref(); - - std::string command = remoteProgram + " --stdio"; - if (remoteStore.get() != "") - command += " --store " + shellEscape(remoteStore.get()); - for (auto & arg : extraRemoteProgramArgs) - command += " " + shellEscape(arg); - - conn->sshConn = master.startCommand(command); + Strings command = remoteProgram.get(); + command.push_back("--stdio"); + if (remoteStore.get() != "") { + command.push_back("--store"); + command.push_back(remoteStore.get()); + } + command.insert(command.end(), + extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); + conn->sshConn = master.startCommand(std::move(command)); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 5c8d6a5042b..30fe73adb09 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -52,7 +52,8 @@ bool SSHMaster::isMasterRunning() { return res.first == 0; } -std::unique_ptr SSHMaster::startCommand(const std::string & command) +std::unique_ptr SSHMaster::startCommand( + Strings && command, Strings && extraSshArgs) { Path socketPath = startMaster(); @@ -84,18 +85,19 @@ std::unique_ptr SSHMaster::startCommand(const std::string Strings args; - if (fakeSSH) { - args = { "bash", "-c" }; - } else { + if (!fakeSSH) { args = { "ssh", host.c_str(), "-x" }; addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); if (verbosity >= lvlChatty) args.push_back("-v"); + args.splice(args.end(), std::move(extraSshArgs)); + args.push_back("--"); } - args.push_back(command); + args.splice(args.end(), std::move(command)); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); // could not exec ssh/bash diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index bfcd6f21cea..08bb43dfabd 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -41,7 +41,16 @@ public: AutoCloseFD out, in; }; - std::unique_ptr startCommand(const std::string & command); + /** + * @param command The command (arg vector) to execute. + * + * @param extraSShArgs Extra args to pass to SSH (not the command to + * execute). Will not be used when "fake SSHing" to the local + * machine. + */ + std::unique_ptr startCommand( + Strings && command, + Strings && extraSshArgs = {}); Path startMaster(); }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7f35e74afaa..4238cbbf5b5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,4 +1,4 @@ -#include "crypto.hh" +#include "signature/local-keys.hh" #include "source-accessor.hh" #include "globals.hh" #include "derived-path.hh" @@ -65,85 +65,13 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -/* Store paths have the following form: - - = /- - - where - - = the location of the Nix store, usually /nix/store - - = a human readable name for the path, typically obtained - from the name attribute of the derivation, or the name of the - source file from which the store path is created. For derivation - outputs other than the default "out" output, the string "-" - is suffixed to . - - = base-32 representation of the first 160 bits of a SHA-256 - hash of ; the hash part of the store name - - = the string ":sha256:

::"; - note that it includes the location of the store as well as the - name to make sure that changes to either of those are reflected - in the hash (e.g. you won't get /nix/store/-name1 and - /nix/store/-name2 with equal hash parts). - - = one of: - "text:::..." - for plain text files written to the store using - addTextToStore(); ... are the store paths referenced - by this path, in the form described by - "source:::...::self" - for paths copied to the store using addToStore() when recursive - = true and hashAlgo = "sha256". Just like in the text case, we - can have the store paths referenced by the path. - Additionally, we can have an optional :self label to denote self - reference. - "output:" - for either the outputs created by derivations, OR paths copied - to the store using addToStore() with recursive != true or - hashAlgo != "sha256" (in that case "source" is used; it's - silly, but it's done that way for compatibility). is the - name of the output (usually, "out"). - -

= base-16 representation of a SHA-256 hash of - - = - if = "text:...": - the string written to the resulting store path - if = "source:...": - the serialisation of the path from which this store path is - copied, as returned by hashPath() - if = "output:": - for non-fixed derivation outputs: - the derivation (see hashDerivationModulo() in - primops.cc) - for paths copied by addToStore() or produced by fixed-output - derivations: - the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes - = "md5", "sha1" or "sha256" - = base-16 representation of the path or flat hash of - the contents of the path (or expected contents of the - path for fixed-output derivations) - - Note that since an output derivation has always type output, while - something added by addToStore can have type output or source depending - on the hash, this means that the same input can be hashed differently - if added to the store via addToStore or via a derivation, in the sha256 - recursive case. - - It would have been nicer to handle fixed-output derivations under - "source", e.g. have something like "source:", but we're - stuck with this for now... - - The main reason for this way of computing names is to prevent name - collisions (for security). For instance, it shouldn't be feasible - to come up with a derivation whose output path collides with the - path for a copied source. The former would have a starting with - "output:out:", while the latter would have a starting with - "source:". +/* +The exact specification of store paths is in `protocols/store-path.md` +in the Nix manual. These few functions implement that specification. + +If changes to these functions go beyond mere implementation changes i.e. +also update the user-visible behavior, please update the specification +to match. */ @@ -194,7 +122,10 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(info.references.size() == 0); + if (!info.references.empty()) { + throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.", + name); + } return makeStorePath("output:out", hashString(HashAlgorithm::SHA256, "fixed:out:" @@ -205,25 +136,19 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const -{ - assert(info.hash.algo == HashAlgorithm::SHA256); - return makeStorePath( - makeType(*this, "text", StoreReferences { - .others = info.references, - .self = false, - }), - info.hash, - name); -} - - StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(name, ti); + assert(ti.hash.algo == HashAlgorithm::SHA256); + return makeStorePath( + makeType(*this, "text", StoreReferences { + .others = ti.references, + .self = false, + }), + ti.hash, + name); }, [&](const FixedOutputInfo & foi) { return makeFixedOutputPath(name, foi); @@ -232,54 +157,45 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - const StorePathSet & references) const -{ - HashSink sink(hashAlgo); - dump.drainInto(sink); - auto h = sink.finish().first; - FixedOutputInfo caInfo { - .method = method, - .hash = h, - .references = {}, - }; - return std::make_pair(makeFixedOutputPath(name, caInfo), h); -} - - -StorePath StoreDirConfig::computeStorePathForText( +std::pair StoreDirConfig::computeStorePath( std::string_view name, - std::string_view s, - const StorePathSet & references) const + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter) const { - return makeTextPath(name, TextInfo { - .hash = hashString(HashAlgorithm::SHA256, s), - .references = references, - }); + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; + return { + makeFixedOutputPathFromCA( + name, + ContentAddressWithReferences::fromParts( + method, + h, + { + .others = references, + .self = false, + })), + h, + }; } StorePath Store::addToStore( - std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) { - Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - readFile(srcPath, sink); + dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); + return addToStoreFromDump(*source, name, method, hashAlgo, references, repair); } void Store::addMultipleToStore( @@ -404,9 +320,13 @@ digraph graphname { fileSink -> caHashSink } */ -ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashAlgorithm hashAlgo, - std::optional expectedCAHash) +ValidPathInfo Store::addToStoreSlow( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, HashAlgorithm hashAlgo, + const StorePathSet & references, + std::optional expectedCAHash) { HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink caHashSink { hashAlgo }; @@ -425,17 +345,19 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, srcPath. The fact that we use scratchpadSink as a temporary buffer here is an implementation detail. */ auto fileSource = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + accessor.dumpPath(srcPath, scratchpadSink); }); /* tapped provides the same data as fileSource, but we also write all the information to narSink. */ TeeSource tapped { *fileSource, narSink }; - NullParseSink blank; - auto & parseSink = method == FileIngestionMethod::Flat - ? (ParseSink &) fileSink - : (ParseSink &) blank; + NullFileSystemObjectSink blank; + auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat + ? (FileSystemObjectSink &) fileSink + : method.getFileIngestionMethod() == FileIngestionMethod::Recursive + ? (FileSystemObjectSink &) blank + : (abort(), (FileSystemObjectSink &)*(FileSystemObjectSink *)nullptr); // handled both cases /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -452,21 +374,24 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); + ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = hash, - .references = {}, - }, + ContentAddressWithReferences::fromParts( + method, + hash, + { + .others = references, + .self = false, + }), narHash, }; info.narSize = narSize; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + accessor.dumpPath(srcPath, scratchpadSink); }); addToStore(info, *source); } @@ -689,38 +614,56 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) } -void Store::queryPathInfo(const StorePath & storePath, - Callback> callback) noexcept +std::optional> Store::queryPathInfoFromClientCache(const StorePath & storePath) { auto hashPart = std::string(storePath.hashPart()); - try { - { - auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); - if (res && res->isKnownNow()) { - stats.narInfoReadAverted++; - if (!res->didExist()) - throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); - return callback(ref(res->value)); - } + { + auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); + if (res && res->isKnownNow()) { + stats.narInfoReadAverted++; + if (res->didExist()) + return std::make_optional(res->value); + else + return std::make_optional(nullptr); } + } - if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); - if (res.first != NarInfoDiskCache::oUnknown) { - stats.narInfoReadAverted++; - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(std::string(storePath.to_string()), - res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); - if (res.first == NarInfoDiskCache::oInvalid || - !goodStorePath(storePath, res.second->path)) - throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); - } - return callback(ref(res.second)); + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(std::string(storePath.to_string()), + res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); + if (res.first == NarInfoDiskCache::oInvalid || + !goodStorePath(storePath, res.second->path)) + return std::make_optional(nullptr); } + assert(res.second); + return std::make_optional(res.second); } + } + + return std::nullopt; +} + + +void Store::queryPathInfo(const StorePath & storePath, + Callback> callback) noexcept +{ + auto hashPart = std::string(storePath.hashPart()); + try { + auto r = queryPathInfoFromClientCache(storePath); + if (r.has_value()) { + std::shared_ptr & info = *r; + if (info) + return callback(ref(info)); + else + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); + } } catch (...) { return callback.rethrow(); } auto callbackPtr = std::make_shared(std::move(callback)); @@ -832,7 +775,7 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { std::vector subs; - for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); + for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); @@ -984,6 +927,11 @@ void copyStorePath( RepairFlag repair, CheckSigsFlag checkSigs) { + /* Bail out early (before starting a download from srcStore) if + dstStore already has this path. */ + if (!repair && dstStore.isValidPath(storePath)) + return; + auto srcUri = srcStore.getUri(); auto dstUri = dstStore.getUri(); auto storePathS = srcStore.printStorePath(storePath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2c883ce97ec..5163070b2e6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -13,6 +13,7 @@ #include "path-info.hh" #include "repair-flag.hh" #include "store-dir-config.hh" +#include "source-path.hh" #include #include @@ -107,7 +108,7 @@ struct StoreConfig : public StoreDirConfig StoreConfig() = delete; - StringSet getDefaultSystemFeatures(); + static StringSet getDefaultSystemFeatures(); virtual ~StoreConfig() { } @@ -281,6 +282,16 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; + /** + * Version of queryPathInfo() that only queries the local narinfo cache and not + * the actual store. + * + * @return `std::nullopt` if nothing is known about the path in the local narinfo cache. + * @return `std::make_optional(nullptr)` if the path is known to not exist. + * @return `std::make_optional(validPathInfo)` if the path is known to exist. + */ + std::optional> queryPathInfoFromClientCache(const StorePath & path); + /** * Query the information about a realisation. */ @@ -427,22 +438,28 @@ public: * libutil/archive.hh). */ virtual StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + PathFilter & filter = defaultPathFilter, + RepairFlag repair = NoRepair); /** * Copy the contents of a path to the store and register the * validity the resulting path, using a constant amount of * memory. */ - ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - std::optional expectedCAHash = {}); + ValidPathInfo addToStoreSlow( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + std::optional expectedCAHash = {}); /** * Like addToStore(), but the contents of the path are contained @@ -453,19 +470,12 @@ public: * * \todo remove? */ - virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) - { unsupported("addToStoreFromDump"); } - - /** - * Like addToStore, but the contents written to the output path is a - * regular file containing the given string. - */ - virtual StorePath addTextToStore( + virtual StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, - const StorePathSet & references, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; /** diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 8dafca096b9..7ca8c2665c3 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -86,41 +86,20 @@ struct StoreDirConfig : public Config StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. + * Read-only variant of addToStore(). It returns the store + * path for the given file sytem object. */ - std::pair computeStorePathFromDump( - Source & dump, + std::pair computeStorePath( std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - const StorePathSet & references = {}) const; - - /** - * Preparatory part of addTextToStore(). - * - * !!! Computation of the path should take the references given to - * addTextToStore() into account, otherwise we have a (relatively - * minor) security hole: a caller can register a source file with - * bogus references. If there are too many references, the path may - * not be garbage collected when it has to be (not really a problem, - * the caller could create a root anyway), or it may be garbage - * collected when it shouldn't be (more serious). - * - * Hashing the references would solve this (bogus references would - * simply yield a different store path, so other users wouldn't be - * affected), but it has some backwards compatibility issues (the - * hashing scheme changes), so I'm not doing that for now. - */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; + const StorePathSet & references = {}, + PathFilter & filter = defaultPathFilter) const; }; } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 2a379e75e31..a50259d240f 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -7,6 +7,7 @@ #include "archive.hh" #include "path-info.hh" +#include #include namespace nix { @@ -47,6 +48,31 @@ void WorkerProto::Serialise>::write(const StoreDirCon } +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto tag = readNum(conn.from); + switch (tag) { + case 0: + return std::nullopt; + case 1: + return std::optional{std::chrono::microseconds(readNum(conn.from))}; + default: + throw Error("Invalid optional tag from remote"); + } +} + +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optDuration) +{ + if (!optDuration.has_value()) { + conn.to << uint8_t{0}; + } else { + conn.to + << uint8_t{1} + << optDuration.value().count(); + } +} + + DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -110,6 +136,10 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto >> res.startTime >> res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + res.cpuUser = WorkerProto::Serialise>::read(store, conn); + res.cpuSystem = WorkerProto::Serialise>::read(store, conn); + } if (GET_PROTOCOL_MINOR(conn.version) >= 28) { auto builtOutputs = WorkerProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) @@ -132,6 +162,10 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wo << res.startTime << res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + WorkerProto::write(store, conn, res.cpuUser); + WorkerProto::write(store, conn, res.cpuSystem); + } if (GET_PROTOCOL_MINOR(conn.version) >= 28) { DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c269142896c..91d277b7705 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "common-protocol.hh" namespace nix { @@ -9,7 +11,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 36) +#define PROTOCOL_VERSION (1 << 8 | 37) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -214,6 +216,8 @@ template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_WORKER_SERIALISER(std::optional); +template<> +DECLARE_WORKER_SERIALISER(std::optional); template DECLARE_WORKER_SERIALISER(std::vector); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 465df2073d2..351ee094b88 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -77,20 +77,20 @@ void SourceAccessor::dumpPath( std::string name(i.first); size_t pos = i.first.find(caseHackSuffix); if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); + debug("removing case hack suffix from '%s'", path / i.first); name.erase(pos); } if (!unhacked.emplace(name, i.first).second) throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); + (path / unhacked[name]), + (path / i.first)); } else unhacked.emplace(i.first, i.first); for (auto & i : unhacked) - if (filter((path + i.first).abs())) { + if (filter((path / i.first).abs())) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); + dump(path / i.second); sink << ")"; } } @@ -110,8 +110,8 @@ void SourceAccessor::dumpPath( time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - PosixSourceAccessor accessor; - accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path); + accessor.dumpPath(canonPath, sink, filter); return accessor.mtime; } @@ -133,21 +133,21 @@ static SerialisationError badArchive(const std::string & s) } -static void parseContents(ParseSink & sink, Source & source, const Path & path) +static void parseContents(CreateRegularFileSink & sink, Source & source) { uint64_t size = readLongLong(source); sink.preallocateContents(size); uint64_t left = size; - std::vector buf(65536); + std::array buf; while (left) { checkInterrupt(); auto n = buf.size(); if ((uint64_t)n > left) n = left; source(buf.data(), n); - sink.receiveContents({buf.data(), n}); + sink({buf.data(), n}); left -= n; } @@ -164,100 +164,112 @@ struct CaseInsensitiveCompare }; -static void parse(ParseSink & sink, Source & source, const Path & path) +static void parse(FileSystemObjectSink & sink, Source & source, const Path & path) { std::string s; s = readString(source); if (s != "(") throw badArchive("expected open tag"); - enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; - std::map names; - while (1) { + auto getString = [&]() { checkInterrupt(); + return readString(source); + }; - s = readString(source); + // For first iteration + s = getString(); + + while (1) { if (s == ")") { break; } else if (s == "type") { - if (type != tpUnknown) - throw badArchive("multiple type fields"); - std::string t = readString(source); + std::string t = getString(); if (t == "regular") { - type = tpRegular; - sink.createRegularFile(path); + sink.createRegularFile(path, [&](auto & crf) { + while (1) { + s = getString(); + + if (s == "contents") { + parseContents(crf, source); + } + + else if (s == "executable") { + auto s2 = getString(); + if (s2 != "") throw badArchive("executable marker has non-empty value"); + crf.isExecutable(); + } + + else break; + } + }); } else if (t == "directory") { sink.createDirectory(path); - type = tpDirectory; - } - else if (t == "symlink") { - type = tpSymlink; - } + while (1) { + s = getString(); + + if (s == "entry") { + std::string name, prevName; + + s = getString(); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + s = getString(); + + if (s == ")") { + break; + } else if (s == "name") { + name = getString(); + if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) + throw Error("NAR contains invalid file name '%1%'", name); + if (name <= prevName) + throw Error("NAR directory is not sorted"); + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + debug("case collision between '%1%' and '%2%'", i->first, name); + name += caseHackSuffix; + name += std::to_string(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (name.empty()) throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else + throw badArchive("unknown field " + s); + } + } - else throw badArchive("unknown file type " + t); + else break; + } + } - } + else if (t == "symlink") { + s = getString(); - else if (s == "contents" && type == tpRegular) { - parseContents(sink, source, path); - sink.closeRegularFile(); - } + if (s != "target") + throw badArchive("expected 'target' got " + s); - else if (s == "executable" && type == tpRegular) { - auto s = readString(source); - if (s != "") throw badArchive("executable marker has non-empty value"); - sink.isExecutable(); - } + std::string target = getString(); + sink.createSymlink(path, target); - else if (s == "entry" && type == tpDirectory) { - std::string name, prevName; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } else if (s == "name") { - name = readString(source); - if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) - throw Error("NAR contains invalid file name '%1%'", name); - if (name <= prevName) - throw Error("NAR directory is not sorted"); - prevName = name; - if (archiveSettings.useCaseHack) { - auto i = names.find(name); - if (i != names.end()) { - debug("case collision between '%1%' and '%2%'", i->first, name); - name += caseHackSuffix; - name += std::to_string(++i->second); - } else - names[name] = 0; - } - } else if (s == "node") { - if (name.empty()) throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else - throw badArchive("unknown field " + s); + // for the next iteration + s = getString(); } - } - else if (s == "target" && type == tpSymlink) { - std::string target = readString(source); - sink.createSymlink(path, target); + else throw badArchive("unknown file type " + t); + } else @@ -266,7 +278,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } -void parseDump(ParseSink & sink, Source & source) +void parseDump(FileSystemObjectSink & sink, Source & source) { std::string version; try { @@ -294,7 +306,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - NullParseSink parseSink; /* just parse the NAR */ + NullFileSystemObjectSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf8ee89118..28c63bb8550 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -73,7 +73,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -void parseDump(ParseSink & sink, Source & source); +void parseDump(FileSystemObjectSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index e2668c673d1..8996cbe5b00 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -304,7 +304,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) cmdline.push_back(*pos); } - } catch (SysError &) { } + } catch (SystemError &) { } } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { @@ -557,7 +557,7 @@ Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashF assert(*hf == nix::HashFormat::SRI); return Flag{ .longName = std::move(longName), - .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'", + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.", .labels = {"hash-format"}, .handler = {[hf](std::string s) { *hf = parseHashFormat(s); @@ -569,7 +569,7 @@ Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashF Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional * ohf) { return Flag{ .longName = std::move(longName), - .description = "hash format ('base16', 'nix32', 'base64', 'sri').", + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).", .labels = {"hash-format"}, .handler = {[ohf](std::string s) { *ohf = std::optional{parseHashFormat(s)}; @@ -589,7 +589,7 @@ Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * h { return Flag{ .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).", .labels = {"hash-algo"}, .handler = {[ha](std::string s) { *ha = parseHashAlgo(s); @@ -602,7 +602,7 @@ Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional< { return Flag{ .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", + .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.", .labels = {"hash-algo"}, .handler = {[oha](std::string s) { *oha = std::optional{parseHashAlgo(s)}; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 18b0ae5834a..6c9c48065a4 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -177,7 +177,13 @@ protected: std::optional experimentalFeature; static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); + static Flag mkHashAlgoFlag(HashAlgorithm * ha) { + return mkHashAlgoFlag("hash-algo", ha); + } static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); + static Flag mkHashAlgoOptFlag(std::optional * oha) { + return mkHashAlgoOptFlag("hash-algo", oha); + } static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf); static Flag mkHashFormatOptFlag(std::string && longName, std::optional * ohf); }; diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 1e465f1f699..1223ba33c5b 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -6,11 +6,11 @@ namespace nix { CanonPath CanonPath::root = CanonPath("/"); CanonPath::CanonPath(std::string_view raw) - : path(absPath((Path) raw, "/")) + : path(absPath(raw, "/")) { } CanonPath::CanonPath(std::string_view raw, const CanonPath & root) - : path(absPath((Path) raw, root.abs())) + : path(absPath(raw, root.abs())) { } CanonPath::CanonPath(const std::vector & elems) @@ -20,11 +20,6 @@ CanonPath::CanonPath(const std::vector & elems) push(s); } -CanonPath CanonPath::fromCwd(std::string_view path) -{ - return CanonPath(unchecked_t(), absPath((Path) path)); -} - std::optional CanonPath::parent() const { if (isRoot()) return std::nullopt; @@ -63,7 +58,7 @@ void CanonPath::extend(const CanonPath & x) path += x.abs(); } -CanonPath CanonPath::operator + (const CanonPath & x) const +CanonPath CanonPath::operator / (const CanonPath & x) const { auto res = *this; res.extend(x); @@ -78,7 +73,7 @@ void CanonPath::push(std::string_view c) path += c; } -CanonPath CanonPath::operator + (std::string_view c) const +CanonPath CanonPath::operator / (std::string_view c) const { auto res = *this; res.push(c); diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 6aff4ec0d74..2f8ff381edf 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -52,8 +52,6 @@ public: */ CanonPath(const std::vector & elems); - static CanonPath fromCwd(std::string_view path = "."); - static CanonPath root; /** @@ -88,6 +86,13 @@ public: std::string_view rel() const { return ((std::string_view) path).substr(1); } + const char * rel_c_str() const + { + auto cs = path.c_str(); + assert(cs[0]); // for safety if invariant is broken + return &cs[1]; + } + struct Iterator { std::string_view remaining; @@ -183,14 +188,14 @@ public: /** * Concatenate two paths. */ - CanonPath operator + (const CanonPath & x) const; + CanonPath operator / (const CanonPath & x) const; /** * Add a path component to this one. It must not contain any slashes. */ void push(std::string_view c); - CanonPath operator + (std::string_view c) const; + CanonPath operator / (std::string_view c) const; /** * Check whether access to this path is allowed, which is the case diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index 4c2bf31ffe1..de83b5ad107 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -95,7 +95,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) using namespace std::string_literals; warn("killing stray builder process %d (%s)...", pid, trim(replaceStrings(cmdline, "\0"s, " "))); - } catch (SysError &) { + } catch (SystemError &) { } } // FIXME: pid wraparound diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index a4d20a675a1..cbc2bb4fd72 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -13,9 +13,9 @@ #define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ __VA_OPT__(const MY_TYPE * me = this;) \ - auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + auto fields1 = std::tie( __VA_ARGS__ ); \ __VA_OPT__(me = &other;) \ - auto fields2 = std::make_tuple( __VA_ARGS__ ); \ + auto fields2 = std::tie( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ diff --git a/src/libutil/config.cc b/src/libutil/config.cc index a3310f4ecb0..37f5b50c7b3 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -124,7 +124,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p try { std::string includedContents = readFile(path); applyConfigInner(includedContents, p, parsedContents); - } catch (SysError &) { + } catch (SystemError &) { // TODO: Do we actually want to ignore this? Or is it better to fail? } } else if (!ignoreMissing) { diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 352a6a0fb36..47aa137d875 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -1,3 +1,6 @@ +#include +#include + #include "current-process.hh" #include "namespaces.hh" #include "util.hh" @@ -49,20 +52,27 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -#if __linux__ rlim_t savedStackSize = 0; -#endif -void setStackSize(size_t stackSize) +void setStackSize(rlim_t stackSize) { - #if __linux__ struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); + limit.rlim_cur = std::min(stackSize, limit.rlim_max); + if (setrlimit(RLIMIT_STACK, &limit) != 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", + savedStackSize, + stackSize, + limit.rlim_max, + std::strerror(errno) + ).str() + ); + } } - #endif } void restoreProcessContext(bool restoreMounts) @@ -72,7 +82,6 @@ void restoreProcessContext(bool restoreMounts) restoreMountNamespace(); } - #if __linux__ if (savedStackSize) { struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0) { @@ -80,7 +89,6 @@ void restoreProcessContext(bool restoreMounts) setrlimit(RLIMIT_STACK, &limit); } } - #endif } diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh index 826d6fe2006..444c717d1fc 100644 --- a/src/libutil/current-process.hh +++ b/src/libutil/current-process.hh @@ -2,6 +2,7 @@ ///@file #include +#include #include "types.hh" @@ -16,7 +17,7 @@ unsigned int getMaxCPU(); /** * Change the stack size. */ -void setStackSize(size_t stackSize); +void setStackSize(rlim_t stackSize); /** * Restore the original inherited Unix process context (such as signal diff --git a/src/libutil/english.cc b/src/libutil/english.cc new file mode 100644 index 00000000000..8c93c915662 --- /dev/null +++ b/src/libutil/english.cc @@ -0,0 +1,18 @@ +#include "english.hh" + +namespace nix { + +std::ostream & pluralize( + std::ostream & output, + unsigned int count, + const std::string_view single, + const std::string_view plural) +{ + if (count == 1) + output << "1 " << single; + else + output << count << " " << plural; + return output; +} + +} diff --git a/src/libutil/english.hh b/src/libutil/english.hh new file mode 100644 index 00000000000..9c6c9357174 --- /dev/null +++ b/src/libutil/english.hh @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace nix { + +/** + * Pluralize a given value. + * + * If `count == 1`, prints `1 {single}` to `output`, otherwise prints `{count} {plural}`. + */ +std::ostream & pluralize( + std::ostream & output, + unsigned int count, + const std::string_view single, + const std::string_view plural); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 72c346cb583..4a9efc0b503 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -2,6 +2,7 @@ #include "environment-variables.hh" #include "signals.hh" #include "terminal.hh" +#include "position.hh" #include #include @@ -10,7 +11,7 @@ namespace nix { -void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) +void BaseError::addTrace(std::shared_ptr && e, HintFmt hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } @@ -36,63 +37,41 @@ const std::string & BaseError::calcWhat() const std::optional ErrorInfo::programName = std::nullopt; -std::ostream & operator <<(std::ostream & os, const hintformat & hf) +std::ostream & operator <<(std::ostream & os, const HintFmt & hf) { return os << hf.str(); } -std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) -{ - pos.print(str); - str << ":" << pos.line; - if (pos.column > 0) - str << ":" << pos.column; - return str; -} - -std::optional AbstractPos::getCodeLines() const +/** + * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. + */ +inline bool operator<(const Trace& lhs, const Trace& rhs) { - if (line == 0) - return std::nullopt; - - if (auto source = getSource()) { - - std::istringstream iss(*source); - // count the newlines. - int count = 0; - std::string curLine; - int pl = line - 1; - - LinesOfCode loc; - - do { - std::getline(iss, curLine); - ++count; - if (count < pl) - ; - else if (count == pl) { - loc.prevLineOfCode = curLine; - } else if (count == pl + 1) { - loc.errLineOfCode = curLine; - } else if (count == pl + 2) { - loc.nextLineOfCode = curLine; - break; - } - - if (!iss.good()) - break; - } while (true); - - return loc; + // `std::shared_ptr` does not have value semantics for its comparison + // functions, so we need to check for nulls and compare the dereferenced + // values here. + if (lhs.pos != rhs.pos) { + if (!lhs.pos) + return true; + if (!rhs.pos) + return false; + if (*lhs.pos != *rhs.pos) + return *lhs.pos < *rhs.pos; } - - return std::nullopt; + // This formats a freshly formatted hint string and then throws it away, which + // shouldn't be much of a problem because it only runs when pos is equal, and this function is + // used for trace printing, which is infrequent. + return std::forward_as_tuple(lhs.hint.str(), lhs.frame) + < std::forward_as_tuple(rhs.hint.str(), rhs.frame); } +inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } +inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } +inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, const std::string & prefix, - const AbstractPos & errPos, + const Pos & errPos, const LinesOfCode & loc) { // previous line of code. @@ -170,13 +149,12 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h * * @return true if a position was printed. */ -static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { bool hasPos = pos && *pos; if (hasPos) { - oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; if (auto loc = pos->getCodeLines()) { - oss << "\n"; printCodeLines(oss, "", *pos, *loc); oss << "\n"; } @@ -186,6 +164,69 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std return hasPos; } +void printTrace( + std::ostream & output, + const std::string_view & indent, + size_t & count, + const Trace & trace) +{ + output << "\n" << "… " << trace.hint.str() << "\n"; + + if (printPosMaybe(output, indent, trace.pos)) + count++; +} + +void printSkippedTracesMaybe( + std::ostream & output, + const std::string_view & indent, + size_t & count, + std::vector & skippedTraces, + std::set tracesSeen) +{ + if (skippedTraces.size() > 0) { + // If we only skipped a few frames, print them out normally; + // messages like "1 duplicate frames omitted" aren't helpful. + if (skippedTraces.size() <= 5) { + for (auto & trace : skippedTraces) { + printTrace(output, indent, count, trace); + } + } else { + output << "\n" << ANSI_WARNING "(" << skippedTraces.size() << " duplicate frames omitted)" ANSI_NORMAL << "\n"; + // Clear the set of "seen" traces after printing a chunk of + // `duplicate frames omitted`. + // + // Consider a mutually recursive stack trace with: + // - 10 entries of A + // - 10 entries of B + // - 10 entries of A + // + // If we don't clear `tracesSeen` here, we would print output like this: + // - 1 entry of A + // - (9 duplicate frames omitted) + // - 1 entry of B + // - (19 duplicate frames omitted) + // + // This would obscure the control flow, which went from A, + // to B, and back to A again. + // + // In contrast, if we do clear `tracesSeen`, the output looks like this: + // - 1 entry of A + // - (9 duplicate frames omitted) + // - 1 entry of B + // - (9 duplicate frames omitted) + // - 1 entry of A + // - (9 duplicate frames omitted) + // + // See: `tests/functional/lang/eval-fail-mutual-recursion.nix` + tracesSeen.clear(); + } + } + // We've either printed each trace in `skippedTraces` normally, or + // printed a chunk of `duplicate frames omitted`. Either way, we've + // processed these traces and can clear them. + skippedTraces.clear(); +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -294,7 +335,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * try { * e->eval(*this, env, v); * if (v.type() != nAttrs) - * throwTypeError("value is %1% while a set was expected", v); + * error("expected a set but found %1%", v); * } catch (Error & e) { * e.addTrace(pos, errorCtx); * throw; @@ -308,7 +349,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * e->eval(*this, env, v); * try { * if (v.type() != nAttrs) - * throwTypeError("value is %1% while a set was expected", v); + * error("expected a set but found %1%", v); * } catch (Error & e) { * e.addTrace(pos, errorCtx); * throw; @@ -334,7 +375,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s bool frameOnly = false; if (!einfo.traces.empty()) { + // Stack traces seen since we last printed a chunk of `duplicate frames + // omitted`. + std::set tracesSeen; + // A consecutive sequence of stack traces that are all in `tracesSeen`. + std::vector skippedTraces; size_t count = 0; + for (const auto & trace : einfo.traces) { if (trace.hint.str().empty()) continue; if (frameOnly && !trace.frame) continue; @@ -344,20 +391,27 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s break; } + if (tracesSeen.count(trace)) { + skippedTraces.push_back(trace); + continue; + } + tracesSeen.insert(trace); + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); + count++; frameOnly = trace.frame; - oss << "\n" << "… " << trace.hint.str() << "\n"; - - if (printPosMaybe(oss, ellipsisIndent, trace.pos)) - count++; + printTrace(oss, ellipsisIndent, count, trace); } + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); oss << "\n" << prefix; } oss << einfo.msg << "\n"; - printPosMaybe(oss, "", einfo.errPos); + printPosMaybe(oss, "", einfo.pos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { @@ -370,4 +424,5 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index c04dcbd77b2..2e5de5d324c 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -25,20 +25,12 @@ #include #include #include +#include #include #include #include -/* Before 4.7, gcc's std::exception uses empty throw() specifiers for - * its (virtual) destructor and what() in c++11 mode, in violation of spec - */ -#ifdef __GNUC__ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) -#define EXCEPTION_NEEDS_THROW_SPEC -#endif -#endif - namespace nix { @@ -62,53 +54,35 @@ struct LinesOfCode { std::optional nextLineOfCode; }; -/** - * An abstract type that represents a location in a source file. - */ -struct AbstractPos -{ - uint32_t line = 0; - uint32_t column = 0; - - /** - * An AbstractPos may be a "null object", representing an unknown position. - * - * Return true if this position is known. - */ - inline operator bool() const { return line != 0; }; - - /** - * Return the contents of the source file. - */ - virtual std::optional getSource() const - { return std::nullopt; }; - - virtual void print(std::ostream & out) const = 0; - - std::optional getCodeLines() const; - - virtual ~AbstractPos() = default; -}; - -std::ostream & operator << (std::ostream & str, const AbstractPos & pos); +struct Pos; void printCodeLines(std::ostream & out, const std::string & prefix, - const AbstractPos & errPos, + const Pos & errPos, const LinesOfCode & loc); struct Trace { - std::shared_ptr pos; - hintformat hint; + std::shared_ptr pos; + HintFmt hint; bool frame; }; +inline bool operator<(const Trace& lhs, const Trace& rhs); +inline bool operator> (const Trace& lhs, const Trace& rhs); +inline bool operator<=(const Trace& lhs, const Trace& rhs); +inline bool operator>=(const Trace& lhs, const Trace& rhs); + struct ErrorInfo { Verbosity level; - hintformat msg; - std::shared_ptr errPos; + HintFmt msg; + std::shared_ptr pos; std::list traces; + /** + * Exit status. + */ + unsigned int status = 1; + Suggestions suggestions; static std::optional programName; @@ -125,31 +99,34 @@ class BaseError : public std::exception protected: mutable ErrorInfo err; + /** + * Cached formatted contents of `err.msg`. + */ mutable std::optional what_; + /** + * Format `err.msg` and set `what_` to the resulting value. + */ const std::string & calcWhat() const; public: - unsigned int status = 1; // exit status - BaseError(const BaseError &) = default; template BaseError(unsigned int status, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(args...) } - , status(status) + : err { .level = lvlError, .msg = HintFmt(args...), .status = status } { } template explicit BaseError(const std::string & fs, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(fs, args...) } + : err { .level = lvlError, .msg = HintFmt(fs, args...) } { } template BaseError(const Suggestions & sug, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(args...), .suggestions = sug } + : err { .level = lvlError, .msg = HintFmt(args...), .suggestions = sug } { } - BaseError(hintformat hint) + BaseError(HintFmt hint) : err { .level = lvlError, .msg = hint } { } @@ -161,28 +138,31 @@ public: : err(e) { } -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return calcWhat().c_str(); } -#else const char * what() const noexcept override { return calcWhat().c_str(); } -#endif - const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } + void withExitStatus(unsigned int status) + { + err.status = status; + } + + void atPos(std::shared_ptr pos) { + err.pos = pos; + } + void pushTrace(Trace trace) { err.traces.push_front(trace); } template - void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) + void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) { - addTrace(std::move(e), hintfmt(std::string(fs), args...)); + addTrace(std::move(e), HintFmt(std::string(fs), args...)); } - void addTrace(std::shared_ptr && e, hintformat hint, bool frame = false); + void addTrace(std::shared_ptr && e, HintFmt hint, bool frame = false); bool hasTrace() const { return !err.traces.empty(); } @@ -200,20 +180,50 @@ MakeError(Error, BaseError); MakeError(UsageError, Error); MakeError(UnimplementedError, Error); -class SysError : public Error +/** + * To use in catch-blocks. + */ +MakeError(SystemError, Error); + +/** + * POSIX system error, created using `errno`, `strerror` friends. + * + * Throw this, but prefer not to catch this, and catch `SystemError` + * instead. This allows implementations to freely switch between this + * and `WinError` without breaking catch blocks. + * + * However, it is permissible to catch this and rethrow so long as + * certain conditions are not met (e.g. to catch only if `errNo = + * EFooBar`). In that case, try to also catch the equivalent `WinError` + * code. + * + * @todo Rename this to `PosixError` or similar. At this point Windows + * support is too WIP to justify the code churn, but if it is finished + * then a better identifier becomes moe worth it. + */ +class SysError : public SystemError { public: int errNo; + /** + * Construct using the explicitly-provided error number. `strerror` + * will be used to try to add additional information to the message. + */ template - SysError(int errNo_, const Args & ... args) - : Error("") + SysError(int errNo, const Args & ... args) + : SystemError(""), errNo(errNo) { - errNo = errNo_; - auto hf = hintfmt(args...); - err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + auto hf = HintFmt(args...); + err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), strerror(errNo)); } + /** + * Construct using the ambient `errno`. + * + * Be sure to not perform another `errno`-modifying operation before + * calling this constructor! + */ template SysError(const Args & ... args) : SysError(errno, args ...) @@ -221,7 +231,9 @@ public: } }; -/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'. +/** + * Throw an exception for the purpose of checking that exception + * handling works; see 'initLibUtil()'. */ void throwExceptionSelfCheck(); diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc new file mode 100644 index 00000000000..6753e0f490b --- /dev/null +++ b/src/libutil/file-content-address.cc @@ -0,0 +1,74 @@ +#include "file-content-address.hh" +#include "archive.hh" + +namespace nix { + +FileIngestionMethod parseFileIngestionMethod(std::string_view input) +{ + if (input == "flat") { + return FileIngestionMethod::Flat; + } else if (input == "nar") { + return FileIngestionMethod::Recursive; + } else { + throw UsageError("Unknown file ingestion method '%s', expect `flat` or `nar`"); + } +} + + +std::string_view renderFileIngestionMethod(FileIngestionMethod method) +{ + switch (method) { + case FileIngestionMethod::Flat: + return "flat"; + case FileIngestionMethod::Recursive: + return "nar"; + default: + abort(); + } +} + + +void dumpPath( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + FileIngestionMethod method, + PathFilter & filter) +{ + switch (method) { + case FileIngestionMethod::Flat: + accessor.readFile(path, sink); + break; + case FileIngestionMethod::Recursive: + accessor.dumpPath(path, sink, filter); + break; + } +} + + +void restorePath( + const Path & path, + Source & source, + FileIngestionMethod method) +{ + switch (method) { + case FileIngestionMethod::Flat: + writeFile(path, source); + break; + case FileIngestionMethod::Recursive: + restorePath(path, source); + break; + } +} + + +HashResult hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashAlgorithm ht, + PathFilter & filter) +{ + HashSink sink { ht }; + dumpPath(accessor, path, sink, method, filter); + return sink.finish(); +} + +} diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh new file mode 100644 index 00000000000..41f23f2af16 --- /dev/null +++ b/src/libutil/file-content-address.hh @@ -0,0 +1,73 @@ +#pragma once +///@file + +#include "source-accessor.hh" +#include "fs-sink.hh" +#include "util.hh" + +namespace nix { + +/** + * An enumeration of the main ways we can serialize file system + * objects. + */ +enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ + Flat = 0, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in + * Nix Archive format and ingest that. + */ + Recursive = 1, +}; + +/** + * Parse a `FileIngestionMethod` by name. Choice of: + * + * - `flat`: `FileIngestionMethod::Flat` + * - `nar`: `FileIngestionMethod::Recursive` + * + * Oppostite of `renderFileIngestionMethod`. + */ +FileIngestionMethod parseFileIngestionMethod(std::string_view input); + +/** + * Render a `FileIngestionMethod` by name. + * + * Oppostite of `parseFileIngestionMethod`. + */ +std::string_view renderFileIngestionMethod(FileIngestionMethod method); + +/** + * Dump a serialization of the given file system object. + */ +void dumpPath( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + FileIngestionMethod method, + PathFilter & filter = defaultPathFilter); + +/** + * Restore a serialization of the given file system object. + * + * @TODO use an arbitrary `FileSystemObjectSink`. + */ +void restorePath( + const Path & path, + Source & source, + FileIngestionMethod method); + +/** + * Compute the hash of the given file system object according to the + * given method. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + */ +HashResult hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashAlgorithm ht, + PathFilter & filter = defaultPathFilter); + +} diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 38dd70c8e4c..55d57e29bfc 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -96,7 +96,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("making file descriptor non-blocking"); } - Finally finally([&]() { + Finally finally([&] { if (!block) { if (fcntl(fd, F_SETFL, saved) == -1) throw SysError("making file descriptor blocking"); @@ -114,7 +114,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("reading from file"); } else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); + else sink({reinterpret_cast(buf.data()), size_t(rd)}); } } @@ -231,7 +231,7 @@ void closeMostFDs(const std::set & exceptions) } } return; - } catch (SysError &) { + } catch (SystemError &) { } #endif diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index c96effff919..9fa1f62dfc0 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -21,9 +21,16 @@ namespace fs = std::filesystem; namespace nix { -Path absPath(Path path, std::optional dir, bool resolveSymlinks) +Path absPath(PathView path, std::optional dir, bool resolveSymlinks) { - if (path[0] != '/') { + std::string scratch; + + if (path.empty() || path[0] != '/') { + // In this case we need to call `canonPath` on a newly-created + // string. We set `scratch` to that string first, and then set + // `path` to `scratch`. This ensures the newly-created string + // lives long enough for the call to `canonPath`, and allows us + // to just accept a `std::string_view`. if (!dir) { #ifdef __GNU__ /* GNU (aka. GNU/Hurd) doesn't have any limitation on path @@ -35,12 +42,13 @@ Path absPath(Path path, std::optional dir, bool resolveSymlinks) if (!getcwd(buf, sizeof(buf))) #endif throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); + scratch = concatStrings(buf, "/", path); #ifdef __GNU__ free(buf); #endif } else - path = concatStrings(*dir, "/", path); + scratch = concatStrings(*dir, "/", path); + path = scratch; } return canonPath(path, resolveSymlinks); } @@ -82,7 +90,7 @@ Path canonPath(PathView path, bool resolveSymlinks) /* Normal component; copy it. */ else { s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { + if (const auto slash = path.find('/'); slash == path.npos) { s += path; path = {}; } else { @@ -108,14 +116,18 @@ Path canonPath(PathView path, bool resolveSymlinks) } } - return s.empty() ? "/" : std::move(s); + if (s.empty()) { + s = "/"; + } + + return s; } Path dirOf(const PathView path) { Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) + if (pos == path.npos) return "."; return pos == 0 ? "/" : Path(path, 0, pos); } @@ -131,7 +143,7 @@ std::string_view baseNameOf(std::string_view path) last -= 1; auto pos = path.rfind('/', last); - if (pos == std::string::npos) + if (pos == path.npos) pos = 0; else pos += 1; @@ -307,7 +319,7 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync) if (!fd) throw SysError("opening file '%1%'", path); - std::vector buf(64 * 1024); + std::array buf; try { while (true) { diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4637507b35b..464efc242fe 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -41,7 +41,7 @@ struct Source; * specified directory, or the current directory otherwise. The path * is also canonicalised. */ -Path absPath(Path path, +Path absPath(PathView path, std::optional dir = {}, bool resolveSymlinks = false); diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ac72e47fbb3..e996f4ba236 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -8,37 +8,64 @@ namespace nix { - -/** - * Inherit some names from other namespaces for convenience. - */ -using boost::format; - - -/** - * A variadic template that does nothing. Useful to call a function - * for all variadic arguments but ignoring the result. - */ -struct nop { template nop(T...) {} }; - - +namespace { /** - * A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - * equivalent to ‘boost::format(format) % a_0 % ... % - * ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - * takes place). + * A helper for writing `boost::format` expressions. + * + * These are equivalent: + * + * ``` + * formatHelper(formatter, a_0, ..., a_n) + * formatter % a_0 % ... % a_n + * ``` + * + * With a single argument, `formatHelper(s)` is a no-op. */ template inline void formatHelper(F & f) -{ -} +{ } template inline void formatHelper(F & f, const T & x, const Args & ... args) { + // Interpolate one argument and then recurse. formatHelper(f % x, args...); } +/** + * Set the correct exceptions for `fmt`. + */ +void setExceptions(boost::format & fmt) +{ + fmt.exceptions( + boost::io::all_error_bits ^ + boost::io::too_many_args_bit ^ + boost::io::too_few_args_bit); +} +} + +/** + * A helper for writing a `boost::format` expression to a string. + * + * These are (roughly) equivalent: + * + * ``` + * fmt(formatString, a_0, ..., a_n) + * (boost::format(formatString) % a_0 % ... % a_n).str() + * ``` + * + * However, when called with a single argument, the string is returned + * unchanged. + * + * If you write code like this: + * + * ``` + * std::cout << boost::format(stringFromUserInput) << std::endl; + * ``` + * + * And `stringFromUserInput` contains formatting placeholders like `%s`, then + * the code will crash at runtime. `fmt` helps you avoid this pitfall. + */ inline std::string fmt(const std::string & s) { return s; @@ -58,68 +85,96 @@ template inline std::string fmt(const std::string & fs, const Args & ... args) { boost::format f(fs); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + setExceptions(f); formatHelper(f, args...); return f.str(); } -// ----------------------------------------------------------------------------- -// format function for hints in errors. same as fmt, except templated values -// are always in yellow. - +/** + * Values wrapped in this struct are printed in magenta. + * + * By default, arguments to `HintFmt` are printed in magenta. To avoid this, + * either wrap the argument in `Uncolored` or add a specialization of + * `HintFmt::operator%`. + */ template -struct yellowtxt +struct Magenta { - yellowtxt(const T &s) : value(s) {} + Magenta(const T &s) : value(s) {} const T & value; }; template -std::ostream & operator<<(std::ostream & out, const yellowtxt & y) +std::ostream & operator<<(std::ostream & out, const Magenta & y) { return out << ANSI_WARNING << y.value << ANSI_NORMAL; } +/** + * Values wrapped in this class are printed without coloring. + * + * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). + */ template -struct normaltxt +struct Uncolored { - normaltxt(const T & s) : value(s) {} + Uncolored(const T & s) : value(s) {} const T & value; }; template -std::ostream & operator<<(std::ostream & out, const normaltxt & y) +std::ostream & operator<<(std::ostream & out, const Uncolored & y) { return out << ANSI_NORMAL << y.value; } -class hintformat +/** + * A wrapper around `boost::format` which colors interpolated arguments in + * magenta by default. + */ +class HintFmt { +private: + boost::format fmt; + public: - hintformat(const std::string & format) : fmt(format) - { - fmt.exceptions(boost::io::all_error_bits ^ - boost::io::too_many_args_bit ^ - boost::io::too_few_args_bit); - } + /** + * Format the given string literally, without interpolating format + * placeholders. + */ + HintFmt(const std::string & literal) + : HintFmt("%s", Uncolored(literal)) + { } + + /** + * Interpolate the given arguments into the format string. + */ + template + HintFmt(const std::string & format, const Args & ... args) + : HintFmt(boost::format(format), args...) + { } - hintformat(const hintformat & hf) + HintFmt(const HintFmt & hf) : fmt(hf.fmt) { } - hintformat(format && fmt) + template + HintFmt(boost::format && fmt, const Args & ... args) : fmt(std::move(fmt)) - { } + { + setExceptions(fmt); + formatHelper(*this, args...); + } template - hintformat & operator%(const T & value) + HintFmt & operator%(const T & value) { - fmt % yellowtxt(value); + fmt % Magenta(value); return *this; } template - hintformat & operator%(const normaltxt & value) + HintFmt & operator%(const Uncolored & value) { fmt % value.value; return *this; @@ -129,25 +184,8 @@ public: { return fmt.str(); } - -private: - format fmt; }; -std::ostream & operator<<(std::ostream & os, const hintformat & hf); - -template -inline hintformat hintfmt(const std::string & fs, const Args & ... args) -{ - hintformat f(fs); - formatHelper(f, args...); - return f; -} - -inline hintformat hintfmt(const std::string & plain_string) -{ - // we won't be receiving any args in this case, so just print the original string - return hintfmt("%s", normaltxt(plain_string)); -} +std::ostream & operator<<(std::ostream & os, const HintFmt & hf); } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 925e6f05dc2..95b6088da9e 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -7,7 +7,7 @@ namespace nix { void copyRecursive( SourceAccessor & accessor, const CanonPath & from, - ParseSink & sink, const Path & to) + FileSystemObjectSink & sink, const Path & to) { auto stat = accessor.lstat(from); @@ -19,16 +19,12 @@ void copyRecursive( case SourceAccessor::tRegular: { - sink.createRegularFile(to); - if (stat.isExecutable) - sink.isExecutable(); - LambdaSink sink2 { - [&](auto d) { - sink.receiveContents(d); - } - }; - accessor.readFile(from, sink2, [&](uint64_t size) { - sink.preallocateContents(size); + sink.createRegularFile(to, [&](CreateRegularFileSink & crf) { + if (stat.isExecutable) + crf.isExecutable(); + accessor.readFile(from, crf, [&](uint64_t size) { + crf.preallocateContents(size); + }); }); break; } @@ -38,7 +34,7 @@ void copyRecursive( sink.createDirectory(to); for (auto & [name, _] : accessor.readDirectory(from)) { copyRecursive( - accessor, from + name, + accessor, from / name, sink, to + "/" + name); break; } @@ -71,20 +67,24 @@ void RestoreSink::createDirectory(const Path & path) throw SysError("creating directory '%1%'", p); }; -void RestoreSink::createRegularFile(const Path & path) -{ - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); -} +struct RestoreRegularFile : CreateRegularFileSink { + AutoCloseFD fd; + + void operator () (std::string_view data) override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; +}; -void RestoreSink::closeRegularFile() +void RestoreSink::createRegularFile(const Path & path, std::function func) { - /* Call close explicitly to make sure the error is checked */ - fd.close(); + Path p = dstPath + path; + RestoreRegularFile crf; + crf.fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!crf.fd) throw SysError("creating file '%1%'", p); + func(crf); } -void RestoreSink::isExecutable() +void RestoreRegularFile::isExecutable() { struct stat st; if (fstat(fd.get(), &st) == -1) @@ -93,7 +93,7 @@ void RestoreSink::isExecutable() throw SysError("fchmod"); } -void RestoreSink::preallocateContents(uint64_t len) +void RestoreRegularFile::preallocateContents(uint64_t len) { if (!restoreSinkSettings.preallocateContents) return; @@ -111,7 +111,7 @@ void RestoreSink::preallocateContents(uint64_t len) #endif } -void RestoreSink::receiveContents(std::string_view data) +void RestoreRegularFile::operator () (std::string_view data) { writeFull(fd.get(), data); } @@ -122,4 +122,32 @@ void RestoreSink::createSymlink(const Path & path, const std::string & target) nix::createSymlink(target, p); } + +void RegularFileSink::createRegularFile(const Path & path, std::function func) +{ + struct CRF : CreateRegularFileSink { + RegularFileSink & back; + CRF(RegularFileSink & back) : back(back) {} + void operator () (std::string_view data) override + { + back.sink(data); + } + void isExecutable() override {} + } crf { *this }; + func(crf); +} + + +void NullFileSystemObjectSink::createRegularFile(const Path & path, std::function func) +{ + struct : CreateRegularFileSink { + void operator () (std::string_view data) override {} + void isExecutable() override {} + } crf; + // Even though `NullFileSystemObjectSink` doesn't do anything, it's important + // that we call the function, to e.g. advance the parser using this + // sink. + func(crf); +} + } diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index bf54b730193..ae577819a25 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -9,18 +9,13 @@ namespace nix { /** - * \todo Fix this API, it sucks. + * Actions on an open regular file in the process of creating it. + * + * See `FileSystemObjectSink::createRegularFile`. */ -struct ParseSink +struct CreateRegularFileSink : Sink { - virtual void createDirectory(const Path & path) = 0; - - virtual void createRegularFile(const Path & path) = 0; - virtual void receiveContents(std::string_view data) = 0; virtual void isExecutable() = 0; - virtual void closeRegularFile() = 0; - - virtual void createSymlink(const Path & path, const std::string & target) = 0; /** * An optimization. By default, do nothing. @@ -28,46 +23,57 @@ struct ParseSink virtual void preallocateContents(uint64_t size) { }; }; + +struct FileSystemObjectSink +{ + virtual ~FileSystemObjectSink() = default; + + virtual void createDirectory(const Path & path) = 0; + + /** + * This function in general is no re-entrant. Only one file can be + * written at a time. + */ + virtual void createRegularFile( + const Path & path, + std::function) = 0; + + virtual void createSymlink(const Path & path, const std::string & target) = 0; +}; + /** - * Recusively copy file system objects from the source into the sink. + * Recursively copy file system objects from the source into the sink. */ void copyRecursive( SourceAccessor & accessor, const CanonPath & sourcePath, - ParseSink & sink, const Path & destPath); + FileSystemObjectSink & sink, const Path & destPath); /** * Ignore everything and do nothing */ -struct NullParseSink : ParseSink +struct NullFileSystemObjectSink : FileSystemObjectSink { void createDirectory(const Path & path) override { } - void receiveContents(std::string_view data) override { } void createSymlink(const Path & path, const std::string & target) override { } - void createRegularFile(const Path & path) override { } - void closeRegularFile() override { } - void isExecutable() override { } + void createRegularFile( + const Path & path, + std::function) override; }; /** * Write files at the given path */ -struct RestoreSink : ParseSink +struct RestoreSink : FileSystemObjectSink { Path dstPath; void createDirectory(const Path & path) override; - void createRegularFile(const Path & path) override; - void receiveContents(std::string_view data) override; - void isExecutable() override; - void closeRegularFile() override; + void createRegularFile( + const Path & path, + std::function) override; void createSymlink(const Path & path, const std::string & target) override; - - void preallocateContents(uint64_t size) override; - -private: - AutoCloseFD fd; }; /** @@ -75,7 +81,7 @@ private: * `receiveContents` to the underlying `Sink`. For anything but a single * file, set `regular = true` so the caller can fail accordingly. */ -struct RegularFileSink : ParseSink +struct RegularFileSink : FileSystemObjectSink { bool regular = true; Sink & sink; @@ -87,19 +93,14 @@ struct RegularFileSink : ParseSink regular = false; } - void receiveContents(std::string_view data) override - { - sink(data); - } - void createSymlink(const Path & path, const std::string & target) override { regular = false; } - void createRegularFile(const Path & path) override { } - void closeRegularFile() override { } - void isExecutable() override { } + void createRegularFile( + const Path & path, + std::function) override; }; } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 296b75628d0..5733531fa33 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -52,24 +52,22 @@ static std::string getString(Source & source, int n) return v; } - -void parse( - ParseSink & sink, +void parseBlob( + FileSystemObjectSink & sink, const Path & sinkPath, Source & source, - std::function hook, + bool executable, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::GitHashing); - auto type = getString(source, 5); - - if (type == "blob ") { - sink.createRegularFile(sinkPath); + sink.createRegularFile(sinkPath, [&](auto & crf) { + if (executable) + crf.isExecutable(); unsigned long long size = std::stoi(getStringUntil(source, 0)); - sink.preallocateContents(size); + crf.preallocateContents(size); unsigned long long left = size; std::string buf; @@ -79,47 +77,91 @@ void parse( checkInterrupt(); buf.resize(std::min((unsigned long long)buf.capacity(), left)); source(buf); - sink.receiveContents(buf); + crf(buf); left -= buf.size(); } - } else if (type == "tree ") { - unsigned long long size = std::stoi(getStringUntil(source, 0)); - unsigned long long left = size; + }); +} + +void parseTree( + FileSystemObjectSink & sink, + const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; - sink.createDirectory(sinkPath); + sink.createDirectory(sinkPath); - while (left) { - std::string perms = getStringUntil(source, ' '); - left -= perms.size(); - left -= 1; + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; - RawMode rawMode = std::stoi(perms, 0, 8); - auto modeOpt = decodeMode(rawMode); - if (!modeOpt) - throw Error("Unknown Git permission: %o", perms); - auto mode = std::move(*modeOpt); + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); - std::string name = getStringUntil(source, '\0'); - left -= name.size(); - left -= 1; + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; - std::string hashs = getString(source, 20); - left -= 20; + std::string hashs = getString(source, 20); + left -= 20; - Hash hash(HashAlgorithm::SHA1); - std::copy(hashs.begin(), hashs.end(), hash.hash); + Hash hash(HashAlgorithm::SHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); - hook(name, TreeEntry { - .mode = mode, - .hash = hash, - }); + hook(name, TreeEntry { + .mode = mode, + .hash = hash, + }); + } +} - if (mode == Mode::Executable) - sink.isExecutable(); - } +ObjectType parseObjectType( + Source & source, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + return ObjectType::Blob; + } else if (type == "tree ") { + return ObjectType::Tree; } else throw Error("input doesn't look like a Git object"); } +void parse( + FileSystemObjectSink & sink, + const Path & sinkPath, + Source & source, + bool executable, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = parseObjectType(source, xpSettings); + + switch (type) { + case ObjectType::Blob: + parseBlob(sink, sinkPath, source, executable, xpSettings); + break; + case ObjectType::Tree: + parseTree(sink, sinkPath, source, hook, xpSettings); + break; + default: + assert(false); + }; +} + std::optional convertMode(SourceAccessor::Type type) { @@ -133,9 +175,9 @@ std::optional convertMode(SourceAccessor::Type type) } -void restore(ParseSink & sink, Source & source, std::function hook) +void restore(FileSystemObjectSink & sink, Source & source, std::function hook) { - parse(sink, "", source, [&](Path name, TreeEntry entry) { + parse(sink, "", source, false, [&](Path name, TreeEntry entry) { auto [accessor, from] = hook(entry.hash); auto stat = accessor->lstat(from); auto gotOpt = convertMode(stat.type); @@ -217,7 +259,7 @@ Mode dump( { Tree entries; for (auto & [name, _] : accessor.readDirectory(path)) { - auto child = path + name; + auto child = path / name; if (!filter(child.abs())) continue; auto entry = hook(child); diff --git a/src/libutil/git.hh b/src/libutil/git.hh index b24b25dd348..d9eb138e165 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -13,12 +13,19 @@ namespace nix::git { +enum struct ObjectType { + Blob, + Tree, + //Commit, + //Tag, +}; + using RawMode = uint32_t; enum struct Mode : RawMode { Directory = 0040000, - Executable = 0100755, Regular = 0100644, + Executable = 0100755, Symlink = 0120000, }; @@ -59,9 +66,34 @@ using Tree = std::map; */ using SinkHook = void(const Path & name, TreeEntry entry); +/** + * Parse the "blob " or "tree " prefix. + * + * @throws if prefix not recognized + */ +ObjectType parseObjectType( + Source & source, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +void parseBlob( + FileSystemObjectSink & sink, const Path & sinkPath, + Source & source, + bool executable, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +void parseTree( + FileSystemObjectSink & sink, const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Helper putting the previous three `parse*` functions together. + */ void parse( - ParseSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const Path & sinkPath, Source & source, + bool executable, std::function hook, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -81,7 +113,7 @@ using RestoreHook = std::pair(Hash); /** * Wrapper around `parse` and `RestoreSink` */ -void restore(ParseSink & sink, Source & source, std::function hook); +void restore(FileSystemObjectSink & sink, Source & source, std::function hook); /** * Dumps a single file to a sink diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 30456ae5c1f..d067da969b0 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -14,6 +14,8 @@ #include #include +#include + namespace nix { static size_t regularHashSize(HashAlgorithm type) { @@ -261,6 +263,13 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo)); } +Hash Hash::random(HashAlgorithm algo) +{ + Hash hash(algo); + randombytes_buf(hash.hash, hash.hashSize); + return hash; +} + Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha) { if (hashStr.empty()) { @@ -367,15 +376,6 @@ HashResult HashSink::currentHash() } -HashResult hashPath( - HashAlgorithm ha, const Path & path, PathFilter & filter) -{ - HashSink sink(ha); - dumpPath(path, sink, filter); - return sink.finish(); -} - - Hash compressHash(const Hash & hash, unsigned int newSize) { Hash h(hash.algo); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 7bed9e2bd2f..f7e8eb265f0 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -5,7 +5,6 @@ #include "serialise.hh" #include "file-system.hh" - namespace nix { @@ -143,6 +142,11 @@ public: } static Hash dummy; + + /** + * @return a random hash with hash algorithm `algo` + */ + static Hash random(HashAlgorithm algo); }; /** @@ -168,14 +172,11 @@ Hash hashString(HashAlgorithm ha, std::string_view s); Hash hashFile(HashAlgorithm ha, const Path & path); /** - * Compute the hash of the given path, serializing as a Nix Archive and - * then hashing that. + * The final hash and the number of bytes digested. * - * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + * @todo Convert to proper struct */ typedef std::pair HashResult; -HashResult hashPath(HashAlgorithm ha, const Path & path, - PathFilter & filter = defaultPathFilter); /** * Compress a hash to the specified number of bytes by cyclically diff --git a/src/libutil/input-accessor.hh b/src/libutil/input-accessor.hh new file mode 100644 index 00000000000..55b7c2f2f84 --- /dev/null +++ b/src/libutil/input-accessor.hh @@ -0,0 +1,27 @@ +#pragma once +///@file + +#include "source-accessor.hh" +#include "ref.hh" +#include "repair-flag.hh" + +namespace nix { + +MakeError(RestrictedPathError, Error); + +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this +{ + std::optional fingerprint; + + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() + { + return std::nullopt; + } + +}; + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 81efaafeca8..200026c1e06 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -4,15 +4,18 @@ libutil_NAME = libnixutil libutil_DIR := $(d) -libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc) libutil_CXXFLAGS += -I src/libutil -libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context $(foreach i, $(wildcard $(d)/args/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) +$(foreach i, $(wildcard $(d)/signature/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/signature, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) - libutil_LDFLAGS += -lcpuid + libutil_LDFLAGS += -lcpuid endif diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 60b0865bf2c..89fbd194a28 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -4,6 +4,8 @@ #include "terminal.hh" #include "util.hh" #include "config.hh" +#include "source-path.hh" +#include "position.hh" #include #include @@ -114,7 +116,7 @@ void writeToStderr(std::string_view s) { try { writeFull(STDERR_FILENO, s, false); - } catch (SysError & e) { + } catch (SystemError & e) { /* Ignore failing writes to stderr. We need to ignore write errors to ensure that cleanup code that logs to stderr runs to completion if the other side of stderr has been closed @@ -136,13 +138,13 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, logger.startActivity(id, lvl, type, s, fields, parent); } -void to_json(nlohmann::json & json, std::shared_ptr pos) +void to_json(nlohmann::json & json, std::shared_ptr pos) { if (pos) { json["line"] = pos->line; json["column"] = pos->column; std::ostringstream str; - pos->print(str); + pos->print(str, true); json["file"] = str.str(); } else { json["line"] = nullptr; @@ -197,7 +199,7 @@ struct JSONLogger : Logger { json["level"] = ei.level; json["msg"] = oss.str(); json["raw_msg"] = ei.msg.str(); - to_json(json, ei.errPos); + to_json(json, ei.pos); if (loggerSettings.showTrace.get() && !ei.traces.empty()) { nlohmann::json traces = nlohmann::json::array(); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 183f2d8e13e..9e81132e354 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -120,6 +120,17 @@ public: { } }; +/** + * A variadic template that does nothing. + * + * Useful to call a function with each argument in a parameter pack. + */ +struct nop +{ + template nop(T...) + { } +}; + ActivityId getCurActivity(); void setCurActivity(const ActivityId activityId); diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 78a4dd29815..880fa61b7f8 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -134,36 +134,43 @@ void MemorySink::createDirectory(const Path & path) throw Error("file '%s' is not a directory", path); }; -void MemorySink::createRegularFile(const Path & path) +struct CreateMemoryRegularFile : CreateRegularFileSink { + File::Regular & regularFile; + + CreateMemoryRegularFile(File::Regular & r) + : regularFile(r) + { } + + void operator () (std::string_view data) override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; +}; + +void MemorySink::createRegularFile(const Path & path, std::function func) { auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); - if (!(r = std::get_if(&f->raw))) + if (auto * rp = std::get_if(&f->raw)) { + CreateMemoryRegularFile crf { *rp }; + func(crf); + } else throw Error("file '%s' is not a regular file", path); } -void MemorySink::closeRegularFile() -{ - r = nullptr; -} - -void MemorySink::isExecutable() +void CreateMemoryRegularFile::isExecutable() { - assert(r); - r->executable = true; + regularFile.executable = true; } -void MemorySink::preallocateContents(uint64_t len) +void CreateMemoryRegularFile::preallocateContents(uint64_t len) { - assert(r); - r->contents.reserve(len); + regularFile.contents.reserve(len); } -void MemorySink::receiveContents(std::string_view data) +void CreateMemoryRegularFile::operator () (std::string_view data) { - assert(r); - r->contents += data; + regularFile.contents += data; } void MemorySink::createSymlink(const Path & path, const std::string & target) diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index b908f3713c0..7a1990d2f72 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -75,7 +75,7 @@ struct MemorySourceAccessor : virtual SourceAccessor /** * Write to a `MemorySourceAccessor` at the given path */ -struct MemorySink : ParseSink +struct MemorySink : FileSystemObjectSink { MemorySourceAccessor & dst; @@ -83,17 +83,11 @@ struct MemorySink : ParseSink void createDirectory(const Path & path) override; - void createRegularFile(const Path & path) override; - void receiveContents(std::string_view data) override; - void isExecutable() override; - void closeRegularFile() override; + void createRegularFile( + const Path & path, + std::function) override; void createSymlink(const Path & path, const std::string & target) override; - - void preallocateContents(uint64_t size) override; - -private: - MemorySourceAccessor::File::Regular * r; }; } diff --git a/src/libutil/position.cc b/src/libutil/position.cc new file mode 100644 index 00000000000..b39a5a1d4b5 --- /dev/null +++ b/src/libutil/position.cc @@ -0,0 +1,112 @@ +#include "position.hh" + +namespace nix { + +Pos::Pos(const Pos * other) +{ + if (!other) { + return; + } + line = other->line; + column = other->column; + origin = std::move(other->origin); +} + +Pos::operator std::shared_ptr() const +{ + return std::make_shared(&*this); +} + +bool Pos::operator<(const Pos &rhs) const +{ + return std::forward_as_tuple(line, column, origin) + < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin); +} + +std::optional Pos::getCodeLines() const +{ + if (line == 0) + return std::nullopt; + + if (auto source = getSource()) { + + std::istringstream iss(*source); + // count the newlines. + int count = 0; + std::string curLine; + int pl = line - 1; + + LinesOfCode loc; + + do { + std::getline(iss, curLine); + ++count; + if (count < pl) + ; + else if (count == pl) { + loc.prevLineOfCode = curLine; + } else if (count == pl + 1) { + loc.errLineOfCode = curLine; + } else if (count == pl + 2) { + loc.nextLineOfCode = curLine; + break; + } + + if (!iss.good()) + break; + } while (true); + + return loc; + } + + return std::nullopt; +} + + +std::optional Pos::getSource() const +{ + return std::visit(overloaded { + [](const std::monostate &) -> std::optional { + return std::nullopt; + }, + [](const Pos::Stdin & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const Pos::String & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const SourcePath & path) -> std::optional { + try { + return path.readFile(); + } catch (Error &) { + return std::nullopt; + } + } + }, origin); +} + +void Pos::print(std::ostream & out, bool showOrigin) const +{ + if (showOrigin) { + std::visit(overloaded { + [&](const std::monostate &) { out << "«none»"; }, + [&](const Pos::Stdin &) { out << "«stdin»"; }, + [&](const Pos::String & s) { out << "«string»"; }, + [&](const SourcePath & path) { out << path; } + }, origin); + out << ":"; + } + out << line; + if (column > 0) + out << ":" << column; +} + +std::ostream & operator<<(std::ostream & str, const Pos & pos) +{ + pos.print(str, true); + return str; +} + +} diff --git a/src/libutil/position.hh b/src/libutil/position.hh new file mode 100644 index 00000000000..a184997ed85 --- /dev/null +++ b/src/libutil/position.hh @@ -0,0 +1,74 @@ +#pragma once +/** + * @file + * + * @brief Pos and AbstractPos + */ + +#include +#include + +#include "source-path.hh" + +namespace nix { + +/** + * A position and an origin for that position (like a source file). + */ +struct Pos +{ + uint32_t line = 0; + uint32_t column = 0; + + struct Stdin { + ref source; + bool operator==(const Stdin & rhs) const + { return *source == *rhs.source; } + bool operator!=(const Stdin & rhs) const + { return *source != *rhs.source; } + bool operator<(const Stdin & rhs) const + { return *source < *rhs.source; } + }; + struct String { + ref source; + bool operator==(const String & rhs) const + { return *source == *rhs.source; } + bool operator!=(const String & rhs) const + { return *source != *rhs.source; } + bool operator<(const String & rhs) const + { return *source < *rhs.source; } + }; + + typedef std::variant Origin; + + Origin origin = std::monostate(); + + Pos() { } + Pos(uint32_t line, uint32_t column, Origin origin) + : line(line), column(column), origin(origin) { } + Pos(Pos & other) = default; + Pos(const Pos & other) = default; + Pos(Pos && other) = default; + Pos(const Pos * other); + + explicit operator bool() const { return line > 0; } + + operator std::shared_ptr() const; + + /** + * Return the contents of the source file. + */ + std::optional getSource() const; + + void print(std::ostream & out, bool showOrigin) const; + + std::optional getCodeLines() const; + + bool operator==(const Pos & rhs) const = default; + bool operator!=(const Pos & rhs) const = default; + bool operator<(const Pos & rhs) const; +}; + +std::ostream & operator<<(std::ostream & str, const Pos & pos); + +} diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 15ff76e59c4..0300de01e6d 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -6,6 +6,33 @@ namespace nix { +PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && root) + : root(std::move(root)) +{ + assert(root.empty() || root.is_absolute()); + displayPrefix = root; +} + +PosixSourceAccessor::PosixSourceAccessor() + : PosixSourceAccessor(std::filesystem::path {}) +{ } + +std::pair PosixSourceAccessor::createAtRoot(const std::filesystem::path & path) +{ + std::filesystem::path path2 = absPath(path.native()); + return { + PosixSourceAccessor { path2.root_path() }, + CanonPath { static_cast(path2.relative_path()) }, + }; +} + +std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) +{ + return root.empty() + ? (std::filesystem::path { path.abs() }) + : root / path.rel(); +} + void PosixSourceAccessor::readFile( const CanonPath & path, Sink & sink, @@ -13,9 +40,11 @@ void PosixSourceAccessor::readFile( { assertNoSymlinks(path); - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + auto ap = makeAbsPath(path); + + AutoCloseFD fd = open(ap.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); if (!fd) - throw SysError("opening file '%1%'", path); + throw SysError("opening file '%1%'", ap.native()); struct stat st; if (fstat(fd.get(), &st) == -1) @@ -25,7 +54,7 @@ void PosixSourceAccessor::readFile( off_t left = st.st_size; - std::vector buf(64 * 1024); + std::array buf; while (left) { checkInterrupt(); ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); @@ -46,7 +75,7 @@ void PosixSourceAccessor::readFile( bool PosixSourceAccessor::pathExists(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::pathExists(path.abs()); + return nix::pathExists(makeAbsPath(path)); } std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) @@ -60,7 +89,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa } std::optional st{std::in_place}; - if (::lstat(path.c_str(), &*st)) { + if (::lstat(makeAbsPath(path).c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) st.reset(); else @@ -95,7 +124,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & { assertNoSymlinks(path); DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { + for (auto & entry : nix::readDirectory(makeAbsPath(path))) { std::optional type; switch (entry.type) { case DT_REG: type = Type::tRegular; break; @@ -110,12 +139,12 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & std::string PosixSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::readLink(path.abs()); + return nix::readLink(makeAbsPath(path)); } -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) { - return path; + return makeAbsPath(path); } void PosixSourceAccessor::assertNoSymlinks(CanonPath path) diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index b2bd3980598..717c8f01779 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -9,6 +9,16 @@ namespace nix { */ struct PosixSourceAccessor : virtual SourceAccessor { + /** + * Optional root path to prefix all operations into the native file + * system. This allows prepending funny things like `C:\` that + * `CanonPath` intentionally doesn't support. + */ + const std::filesystem::path root; + + PosixSourceAccessor(); + PosixSourceAccessor(std::filesystem::path && root); + /** * The most recent mtime seen by lstat(). This is a hack to * support dumpPathAndGetMtime(). Should remove this eventually. @@ -28,7 +38,22 @@ struct PosixSourceAccessor : virtual SourceAccessor std::string readLink(const CanonPath & path) override; - std::optional getPhysicalPath(const CanonPath & path) override; + std::optional getPhysicalPath(const CanonPath & path) override; + + /** + * Create a `PosixSourceAccessor` and `CanonPath` corresponding to + * some native path. + * + * The `PosixSourceAccessor` is rooted as far up the tree as + * possible, (e.g. on Windows it could scoped to a drive like + * `C:\`). This allows more `..` parent accessing to work. + * + * See + * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) + * and + * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). + */ + static std::pair createAtRoot(const std::filesystem::path & path); private: @@ -38,6 +63,8 @@ private: void assertNoSymlinks(CanonPath path); std::optional cachedLstat(const CanonPath & path); + + std::filesystem::path makeAbsPath(const CanonPath & path); }; } diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 91a0ea66fda..f5d584330ab 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -131,7 +131,7 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - Pid pid = startProcess([&]() { + Pid pid = startProcess([&] { if (setuid(uid) == -1) throw SysError("setting uid"); @@ -168,11 +168,12 @@ void killUser(uid_t uid) ////////////////////////////////////////////////////////////////////// +using ChildWrapperFunction = std::function; /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) +static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) { #ifdef __linux__ pid_t pid = allowVfork ? vfork() : fork(); @@ -188,8 +189,8 @@ static pid_t doFork(bool allowVfork, std::function fun) #if __linux__ static int childEntry(void * arg) { - auto main = (std::function *) arg; - (*main)(); + auto & fun = *reinterpret_cast(arg); + fun(); return 1; } #endif @@ -197,7 +198,7 @@ static int childEntry(void * arg) pid_t startProcess(std::function fun, const ProcessOptions & options) { - std::function wrapper = [&]() { + ChildWrapperFunction wrapper = [&] { if (!options.allowVfork) logger = makeSimpleLogger(); try { @@ -225,11 +226,11 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) assert(!(options.cloneFlags & CLONE_VM)); size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + auto stack = static_cast(mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0)); if (stack == MAP_FAILED) throw SysError("allocating stack"); - Finally freeStack([&]() { munmap(stack, stackSize); }); + Finally freeStack([&] { munmap(stack, stackSize); }); pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); #else @@ -308,7 +309,7 @@ void runProgram2(const RunOptions & options) } /* Fork. */ - Pid pid = startProcess([&]() { + Pid pid = startProcess([&] { if (options.environment) replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) @@ -350,7 +351,7 @@ void runProgram2(const RunOptions & options) std::promise promise; - Finally doJoin([&]() { + Finally doJoin([&] { if (writerThread.joinable()) writerThread.join(); }); @@ -358,7 +359,7 @@ void runProgram2(const RunOptions & options) if (source) { in.readSide.close(); - writerThread = std::thread([&]() { + writerThread = std::thread([&] { try { std::vector buf(8 * 1024); while (true) { diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index af5f8304c0d..5d0c3696dbd 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include #include diff --git a/src/libstore/repair-flag.hh b/src/libutil/repair-flag.hh similarity index 100% rename from src/libstore/repair-flag.hh rename to src/libutil/repair-flag.hh diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f465bd0defd..70c16ff0dce 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -53,7 +53,7 @@ void FdSink::writeUnbuffered(std::string_view data) written += data.size(); try { writeFull(fd, data); - } catch (SysError & e) { + } catch (SystemError & e) { _good = false; throw; } @@ -82,7 +82,7 @@ void Source::operator () (std::string_view data) void Source::drainInto(Sink & sink) { std::string s; - std::vector buf(8192); + std::array buf; while (true) { size_t n; try { @@ -132,7 +132,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len) n = ::read(fd, data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } - if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); } read += n; return n; } @@ -448,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintfmt(msg), + .msg = HintFmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -457,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintfmt(readString(source)) + .hint = HintFmt(readString(source)) }); } return Error(std::move(info)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 3f57ce88ba7..d9522566f7d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -153,12 +153,13 @@ struct FdSource : BufferedSource { int fd; size_t read = 0; + BackedStringView endOfFileError{"unexpected end-of-file"}; FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - FdSource(FdSource&&) = default; + FdSource(FdSource &&) = default; - FdSource& operator=(FdSource && s) + FdSource & operator=(FdSource && s) { fd = s.fd; s.fd = -1; diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 4632aa319d8..eaa4ea30e9e 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -179,7 +179,7 @@ std::unique_ptr createInterruptCallback(std::function auto token = interruptCallbacks->nextToken++; interruptCallbacks->callbacks.emplace(token, callback); - auto res = std::make_unique(); + std::unique_ptr res {new InterruptCallbackImpl{}}; res->token = token; return std::unique_ptr(res.release()); diff --git a/src/libstore/crypto.cc b/src/libutil/signature/local-keys.cc similarity index 64% rename from src/libstore/crypto.cc rename to src/libutil/signature/local-keys.cc index 1b705733c30..858b036f550 100644 --- a/src/libstore/crypto.cc +++ b/src/libutil/signature/local-keys.cc @@ -1,13 +1,12 @@ -#include "crypto.hh" +#include "signature/local-keys.hh" + #include "file-system.hh" #include "util.hh" -#include "globals.hh" - #include namespace nix { -static std::pair split(std::string_view s) +BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) @@ -17,10 +16,10 @@ static std::pair split(std::string_view s) Key::Key(std::string_view s) { - auto ss = split(s); + auto ss = BorrowedCryptoValue::parse(s); - name = ss.first; - key = ss.second; + name = ss.name; + key = ss.payload; if (name == "" || key == "") throw Error("secret key is corrupt"); @@ -73,45 +72,34 @@ PublicKey::PublicKey(std::string_view s) throw Error("public key is not valid"); } -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys) +bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) const { - auto ss = split(sig); + auto ss = BorrowedCryptoValue::parse(sig); - auto key = publicKeys.find(std::string(ss.first)); - if (key == publicKeys.end()) return false; + if (ss.name != std::string_view { name }) return false; + + return verifyDetachedAnon(data, ss.payload); +} - auto sig2 = base64Decode(ss.second); +bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const +{ + auto sig2 = base64Decode(sig); if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid"); return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), - (unsigned char *) key->second.key.data()) == 0; + (unsigned char *) key.data()) == 0; } -PublicKeys getDefaultPublicKeys() +bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys) { - PublicKeys publicKeys; - - // FIXME: filter duplicates - - for (auto s : settings.trustedPublicKeys.get()) { - PublicKey key(s); - publicKeys.emplace(key.name, key); - } - - for (auto secretKeyFile : settings.secretKeyFiles.get()) { - try { - SecretKey secretKey(readFile(secretKeyFile)); - publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); - } catch (SysError & e) { - /* Ignore unreadable key files. That's normal in a - multi-user installation. */ - } - } - - return publicKeys; + auto ss = BorrowedCryptoValue::parse(sig); + + auto key = publicKeys.find(std::string(ss.name)); + if (key == publicKeys.end()) return false; + + return key->second.verifyDetachedAnon(data, ss.payload); } } diff --git a/src/libstore/crypto.hh b/src/libutil/signature/local-keys.hh similarity index 51% rename from src/libstore/crypto.hh rename to src/libutil/signature/local-keys.hh index 35216d4709c..4aafc123944 100644 --- a/src/libstore/crypto.hh +++ b/src/libutil/signature/local-keys.hh @@ -7,6 +7,25 @@ namespace nix { +/** + * Except where otherwise noted, Nix serializes keys and signatures in + * the form: + * + * ``` + * : + * ``` + */ +struct BorrowedCryptoValue { + std::string_view name; + std::string_view payload; + + /** + * This splits on the colon, the user can then separated decode the + * Base64 payload separately. + */ + static BorrowedCryptoValue parse(std::string_view); +}; + struct Key { std::string name; @@ -49,21 +68,36 @@ struct PublicKey : Key { PublicKey(std::string_view data); + /** + * @return true iff `sig` and this key's names match, and `sig` is a + * correct signature over `data` using the given public key. + */ + bool verifyDetached(std::string_view data, std::string_view sigs) const; + + /** + * @return true iff `sig` is a correct signature over `data` using the + * given public key. + * + * @param just the Base64 signature itself, not a colon-separated pair of a + * public key name and signature. + */ + bool verifyDetachedAnon(std::string_view data, std::string_view sigs) const; + private: PublicKey(std::string_view name, std::string && key) : Key(name, std::move(key)) { } friend struct SecretKey; }; +/** + * Map from key names to public keys + */ typedef std::map PublicKeys; /** * @return true iff ‘sig’ is a correct signature over ‘data’ using one * of the given public keys. */ -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys); - -PublicKeys getDefaultPublicKeys(); +bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys); } diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc new file mode 100644 index 00000000000..0d26867b54a --- /dev/null +++ b/src/libutil/signature/signer.cc @@ -0,0 +1,23 @@ +#include "signature/signer.hh" +#include "error.hh" + +#include + +namespace nix { + +LocalSigner::LocalSigner(SecretKey && privateKey) + : privateKey(privateKey) + , publicKey(privateKey.toPublicKey()) +{ } + +std::string LocalSigner::signDetached(std::string_view s) const +{ + return privateKey.signDetached(s); +} + +const PublicKey & LocalSigner::getPublicKey() +{ + return publicKey; +} + +} diff --git a/src/libutil/signature/signer.hh b/src/libutil/signature/signer.hh new file mode 100644 index 00000000000..e50170fe29c --- /dev/null +++ b/src/libutil/signature/signer.hh @@ -0,0 +1,61 @@ +#pragma once + +#include "types.hh" +#include "signature/local-keys.hh" + +#include +#include + +namespace nix { + +/** + * An abstract signer + * + * Derive from this class to implement a custom signature scheme. + * + * It is only necessary to implement signature of bytes and provide a + * public key. + */ +struct Signer +{ + virtual ~Signer() = default; + + /** + * Sign the given data, creating a (detached) signature. + * + * @param data data to be signed. + * + * @return the [detached + * signature](https://en.wikipedia.org/wiki/Detached_signature), + * i.e. just the signature itself without a copy of the signed data. + */ + virtual std::string signDetached(std::string_view data) const = 0; + + /** + * View the public key associated with this `Signer`. + */ + virtual const PublicKey & getPublicKey() = 0; +}; + +using Signers = std::map; + +/** + * Local signer + * + * The private key is held in this machine's RAM + */ +struct LocalSigner : Signer +{ + LocalSigner(SecretKey && privateKey); + + std::string signDetached(std::string_view s) const override; + + const PublicKey & getPublicKey() override; + +private: + + SecretKey privateKey; + PublicKey publicKey; +}; + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 4f4ff09c1fb..aff7da09c44 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "canon-path.hh" #include "hash.hh" @@ -119,7 +121,7 @@ struct SourceAccessor * possible. This is only possible for filesystems that are * materialized in the root filesystem. */ - virtual std::optional getPhysicalPath(const CanonPath & path) + virtual std::optional getPhysicalPath(const CanonPath & path) { return std::nullopt; } bool operator == (const SourceAccessor & x) const diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc new file mode 100644 index 00000000000..0f154e77922 --- /dev/null +++ b/src/libutil/source-path.cc @@ -0,0 +1,105 @@ +#include "source-path.hh" + +namespace nix { + +std::string_view SourcePath::baseName() const +{ return path.baseName().value_or("source"); } + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return {accessor, std::move(*p)}; +} + +std::string SourcePath::readFile() const +{ return accessor->readFile(path); } + +bool SourcePath::pathExists() const +{ return accessor->pathExists(path); } + +InputAccessor::Stat SourcePath::lstat() const +{ return accessor->lstat(path); } + +std::optional SourcePath::maybeLstat() const +{ return accessor->maybeLstat(path); } + +InputAccessor::DirEntries SourcePath::readDirectory() const +{ return accessor->readDirectory(path); } + +std::string SourcePath::readLink() const +{ return accessor->readLink(path); } + +void SourcePath::dumpPath( + Sink & sink, + PathFilter & filter) const +{ return accessor->dumpPath(path, sink, filter); } + +std::optional SourcePath::getPhysicalPath() const +{ return accessor->getPhysicalPath(path); } + +std::string SourcePath::to_string() const +{ return accessor->showPath(path); } + +SourcePath SourcePath::operator / (const CanonPath & x) const +{ return {accessor, path / x}; } + +SourcePath SourcePath::operator / (std::string_view c) const +{ return {accessor, path / c}; } + +bool SourcePath::operator==(const SourcePath & x) const +{ + return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); +} + +bool SourcePath::operator!=(const SourcePath & x) const +{ + return std::tie(*accessor, path) != std::tie(*x.accessor, x.path); +} + +bool SourcePath::operator<(const SourcePath & x) const +{ + return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); +} + +SourcePath SourcePath::resolveSymlinks() const +{ + auto res = SourcePath(accessor); + + int linksAllowed = 1024; + + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.path.pop(); + else { + res.path.push(c); + if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", path); + auto target = res.readLink(); + res.path.pop(); + if (hasPrefix(target, "/")) + res.path = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } + } + } + + return res; +} + +std::ostream & operator<<(std::ostream & str, const SourcePath & path) +{ + str << path.to_string(); + return str; +} + +} diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh new file mode 100644 index 00000000000..a2f4ddd1eb9 --- /dev/null +++ b/src/libutil/source-path.hh @@ -0,0 +1,115 @@ +#pragma once +/** + * @file + * + * @brief SourcePath + */ + +#include "ref.hh" +#include "canon-path.hh" +#include "input-accessor.hh" + +namespace nix { + +/** + * An abstraction for accessing source files during + * evaluation. Currently, it's just a wrapper around `CanonPath` that + * accesses files in the regular filesystem, but in the future it will + * support fetching files in other ways. + */ +struct SourcePath +{ + ref accessor; + CanonPath path; + + SourcePath(ref accessor, CanonPath path = CanonPath::root) + : accessor(std::move(accessor)) + , path(std::move(path)) + { } + + std::string_view baseName() const; + + /** + * Construct the parent of this `SourcePath`. Aborts if `this` + * denotes the root. + */ + SourcePath parent() const; + + /** + * If this `SourcePath` denotes a regular file (not a symlink), + * return its contents; otherwise throw an error. + */ + std::string readFile() const; + + /** + * Return whether this `SourcePath` denotes a file (of any type) + * that exists + */ + bool pathExists() const; + + /** + * Return stats about this `SourcePath`, or throw an exception if + * it doesn't exist. + */ + InputAccessor::Stat lstat() const; + + /** + * Return stats about this `SourcePath`, or std::nullopt if it + * doesn't exist. + */ + std::optional maybeLstat() const; + + /** + * If this `SourcePath` denotes a directory (not a symlink), + * return its directory entries; otherwise throw an error. + */ + InputAccessor::DirEntries readDirectory() const; + + /** + * If this `SourcePath` denotes a symlink, return its target; + * otherwise throw an error. + */ + std::string readLink() const; + + /** + * Dump this `SourcePath` to `sink` as a NAR archive. + */ + void dumpPath( + Sink & sink, + PathFilter & filter = defaultPathFilter) const; + + /** + * Return the location of this path in the "real" filesystem, if + * it has a physical location. + */ + std::optional getPhysicalPath() const; + + std::string to_string() const; + + /** + * Append a `CanonPath` to this path. + */ + SourcePath operator / (const CanonPath & x) const; + + /** + * Append a single component `c` to this path. `c` must not + * contain a slash. A slash is implicitly added between this path + * and `c`. + */ + SourcePath operator / (std::string_view c) const; + + bool operator==(const SourcePath & x) const; + bool operator!=(const SourcePath & x) const; + bool operator<(const SourcePath & x) const; + + /** + * Resolve any symlinks in this `SourcePath` (including its + * parents). The result is a `SourcePath` in which no element is a + * symlink. + */ + SourcePath resolveSymlinks() const; +}; + +std::ostream & operator << (std::ostream & str, const SourcePath & path); + +} diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 187b3e94841..3bb6694f88e 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -132,4 +132,66 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } +time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +{ + time_t lastModified = 0; + + for (;;) { + // FIXME: merge with extract_archive + struct archive_entry * entry; + int r = archive_read_next_header(archive.archive, &entry); + if (r == ARCHIVE_EOF) break; + auto path = archive_entry_pathname(entry); + if (!path) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) + warn(archive_error_string(archive.archive)); + else + archive.check(r); + + lastModified = std::max(lastModified, archive_entry_mtime(entry)); + + switch (archive_entry_filetype(entry)) { + + case AE_IFDIR: + parseSink.createDirectory(path); + break; + + case AE_IFREG: { + parseSink.createRegularFile(path, [&](auto & crf) { + if (archive_entry_mode(entry) & S_IXUSR) + crf.isExecutable(); + + while (true) { + std::vector buf(128 * 1024); + auto n = archive_read_data(archive.archive, buf.data(), buf.size()); + if (n < 0) + throw Error("cannot read file '%s' from tarball", path); + if (n == 0) break; + crf(std::string_view { + (const char *) buf.data(), + (size_t) n, + }); + } + }); + + break; + } + + case AE_IFLNK: { + auto target = archive_entry_symlink(entry); + + parseSink.createSymlink(path, target); + + break; + } + + default: + throw Error("file '%s' in tarball has unsupported file type", path); + } + } + + return lastModified; +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 237d18c31a8..6a9c42149b4 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,7 @@ ///@file #include "serialise.hh" +#include "fs-sink.hh" #include namespace nix { @@ -29,4 +30,6 @@ void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); +time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); + } diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index c5e73561739..9a7dfee568f 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -79,6 +79,8 @@ void ThreadPool::process() void ThreadPool::doWork(bool mainThread) { + ReceiveInterrupts receiveInterrupts; + if (!mainThread) interruptCheck = [&]() { return (bool) quit; }; diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 8949461d247..0bcf9040d95 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -1,6 +1,7 @@ #include "file-system.hh" #include "processes.hh" #include "unix-domain-socket.hh" +#include "util.hh" #include #include @@ -38,63 +39,69 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) } -void bind(int fd, const std::string & path) +static void bindConnectProcHelper( + std::string_view operationName, auto && operation, + int fd, const std::string & path) { - unlink(path.c_str()); - struct sockaddr_un addr; addr.sun_family = AF_UNIX; + // Casting between types like these legacy C library interfaces + // require is forbidden in C++. To maintain backwards + // compatibility, the implementation of the bind/connect functions + // contains some hints to the compiler that allow for this + // special case. + auto * psaddr = reinterpret_cast(&addr); + if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); + Pipe pipe; + pipe.create(); + Pid pid = startProcess([&] { + try { + pipe.readSide.close(); + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + writeFull(pipe.writeSide.get(), "0\n"); + } catch (SysError & e) { + writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo)); + } catch (...) { + writeFull(pipe.writeSide.get(), "-1\n"); + } }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); + pipe.writeSide.close(); + auto errNo = string2Int(chomp(drainFD(pipe.readSide.get()))); + if (!errNo || *errNo == -1) + throw Error("cannot %s to socket at '%s'", operationName, path); + else if (*errNo > 0) { + errno = *errNo; + throw SysError("cannot %s to socket at '%s'", operationName, path); + } } else { memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); } } -void connect(int fd, const std::string & path) +void bind(int fd, const std::string & path) { - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; + unlink(path.c_str()); - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } + bindConnectProcHelper("bind", ::bind, fd, path); +} + + +void connect(int fd, const std::string & path) +{ + bindConnectProcHelper("connect", ::connect, fd, path); } } diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 5c5a30dc210..1ddc6a53649 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; @@ -19,13 +19,15 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*"; const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; /// A Git ref (i.e. branch or tag name). /// \todo check that this is correct. -const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; +/// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@+-]*"; extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 57b64d6074c..c6561441d93 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -13,10 +13,10 @@ std::regex revRegex(revRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "((" + schemeRegex + "):" + "((" + schemeNameRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", + + "(?:#(" + fragmentRegex + "))?", std::regex::ECMAScript); std::smatch match; @@ -183,4 +183,12 @@ std::string fixGitURL(const std::string & url) } } +// https://www.rfc-editor.org/rfc/rfc3986#section-3.1 +bool isValidSchemeName(std::string_view s) +{ + static std::regex regex(schemeNameRegex, std::regex::ECMAScript); + + return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 833f546787b..24806bbff81 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -55,4 +55,13 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme); changes absolute paths into file:// URLs. */ std::string fixGitURL(const std::string & url); +/** + * Whether a string is valid as RFC 3986 scheme name. + * Colon `:` is part of the URI; not the scheme name, and therefore rejected. + * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 + * + * Does not check whether the scheme is understood, as that's context-dependent. + */ +bool isValidSchemeName(std::string_view scheme); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5bb3f374ba7..75bb31c9b4e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -19,7 +20,7 @@ void initLibUtil() { // When exception handling fails, the message tends to be printed by the // C++ runtime, followed by an abort. // For example on macOS we might see an error such as - // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. + // libc++abi: terminating with uncaught exception of type nix::SystemError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. bool caught = false; try { throwExceptionSelfCheck(); @@ -28,6 +29,9 @@ void initLibUtil() { } // This is not actually the main point of this check, but let's make sure anyway: assert(caught); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); } ////////////////////////////////////////////////////////////////////// @@ -48,9 +52,9 @@ template C tokenizeString(std::string_view s, std::string_view separato { C result; auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string_view::npos) { + while (pos != s.npos) { auto end = s.find_first_of(separators, pos + 1); - if (end == std::string_view::npos) end = s.size(); + if (end == s.npos) end = s.size(); result.insert(result.end(), std::string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } @@ -65,7 +69,7 @@ template std::vector tokenizeString(std::string_view s, std::string std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); - return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); + return i == s.npos ? "" : std::string(s, 0, i + 1); } @@ -85,7 +89,7 @@ std::string replaceStrings( { if (from.empty()) return res; size_t pos = 0; - while ((pos = res.find(from, pos)) != std::string::npos) { + while ((pos = res.find(from, pos)) != res.npos) { res.replace(pos, from.size(), to); pos += to.size(); } @@ -98,7 +102,7 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) for (auto & i : rewrites) { if (i.first == i.second) continue; size_t j = 0; - while ((j = s.find(i.first, j)) != std::string::npos) + while ((j = s.find(i.first, j)) != s.npos) s.replace(j, i.first.size(), i.second); } return s; @@ -118,12 +122,11 @@ bool hasSuffix(std::string_view s, std::string_view suffix) } -std::string toLower(const std::string & s) +std::string toLower(std::string s) { - std::string r(s); - for (auto & c : r) + for (auto & c : s) c = std::tolower(c); - return r; + return s; } @@ -131,7 +134,7 @@ std::string shellEscape(const std::string_view s) { std::string r; r.reserve(s.size() + 2); - r += "'"; + r += '\''; for (auto & i : s) if (i == '\'') r += "'\\''"; else r += i; r += '\''; @@ -180,7 +183,7 @@ std::string base64Encode(std::string_view s) std::string base64Decode(std::string_view s) { constexpr char npos = -1; - constexpr std::array base64DecodeChars = [&]() { + constexpr std::array base64DecodeChars = [&] { std::array result{}; for (auto& c : result) c = npos; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 27faa4d6d22..11a0431da89 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -180,7 +180,7 @@ bool hasSuffix(std::string_view s, std::string_view suffix); /** * Convert a string to lower case. */ -std::string toLower(const std::string & s); +std::string toLower(std::string s); /** diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 8e9be14c1f9..a372e4b1c7f 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -148,7 +148,7 @@ static void main_nix_build(int argc, char * * argv) args.push_back(word); } } - } catch (SysError &) { } + } catch (SystemError &) { } } struct MyArgs : LegacyArgs, MixEvalArgs @@ -289,7 +289,7 @@ static void main_nix_build(int argc, char * * argv) if (runEnv) setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); - DrvInfos drvs; + PackageInfos drvs; /* Parse the expressions. */ std::vector exprs; @@ -299,7 +299,7 @@ static void main_nix_build(int argc, char * * argv) else for (auto i : left) { if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath(CanonPath::fromCwd()))); + exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath("."))); else { auto absolute = i; try { @@ -307,7 +307,7 @@ static void main_nix_build(int argc, char * * argv) } catch (Error & e) {}; auto [path, outputNames] = parsePathWithOutputs(absolute); if (evalStore->isStorePath(path) && hasSuffix(path, ".drv")) - drvs.push_back(DrvInfo(*state, evalStore, absolute)); + drvs.push_back(PackageInfo(*state, evalStore, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ @@ -350,7 +350,7 @@ static void main_nix_build(int argc, char * * argv) takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs, vRoot ).first); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); getDerivations( *state, v, @@ -383,8 +383,8 @@ static void main_nix_build(int argc, char * * argv) if (drvs.size() != 1) throw UsageError("nix-shell requires a single derivation"); - auto & drvInfo = drvs.front(); - auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath()); + auto & packageInfo = drvs.front(); + auto drv = evalStore->derivationFromPath(packageInfo.requireDrvPath()); std::vector pathsToBuild; RealisedPath::Set pathsToCopy; @@ -400,7 +400,7 @@ static void main_nix_build(int argc, char * * argv) try { auto expr = state->parseExprFromString( "(import {}).bashInteractive", - state->rootPath(CanonPath::fromCwd())); + state->rootPath(".")); Value v; state->eval(expr, v); @@ -527,7 +527,7 @@ static void main_nix_build(int argc, char * * argv) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); - ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); + ParsedDerivation parsedDrv(packageInfo.requireDrvPath(), drv); if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { auto json = structAttrs.value(); @@ -620,10 +620,10 @@ static void main_nix_build(int argc, char * * argv) std::map> drvMap; - for (auto & drvInfo : drvs) { - auto drvPath = drvInfo.requireDrvPath(); + for (auto & packageInfo : drvs) { + auto drvPath = packageInfo.requireDrvPath(); - auto outputName = drvInfo.queryOutputName(); + auto outputName = packageInfo.queryOutputName(); if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d69915e8f82..5e3de20c5c0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { return st.type == InputAccessor::tRegular - || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists()); + || (st.type == InputAccessor::tDirectory && (path / "default.nix").resolveSymlinks().pathExists()); } @@ -116,7 +116,7 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - auto path2 = (path + i).resolveSymlinks(); + auto path2 = (path / i).resolveSymlinks(); InputAccessor::Stat st; try { @@ -184,7 +184,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, std::string systemFilter, Bindings & autoArgs, - const std::string & pathPrefix, DrvInfos & elems) + const std::string & pathPrefix, PackageInfos & elems) { Value vRoot; loadSourceExpr(state, nixExprPath, vRoot); @@ -195,7 +195,7 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, /* Filter out all derivations not applicable to the current system. */ - for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { + for (PackageInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { j = i; j++; if (systemFilter != "*" && i->querySystem() != systemFilter) elems.erase(i); @@ -203,13 +203,13 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, } -static long getPriority(EvalState & state, DrvInfo & drv) +static long getPriority(EvalState & state, PackageInfo & drv) { return drv.queryMetaInt("priority", 0); } -static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) +static long comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) { return getPriority(state, drv2) - getPriority(state, drv1); } @@ -217,7 +217,7 @@ static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) // FIXME: this function is rather slow since it checks a single path // at a time. -static bool isPrebuilt(EvalState & state, DrvInfo & elem) +static bool isPrebuilt(EvalState & state, PackageInfo & elem) { auto path = elem.queryOutPath(); if (state.store->isValidPath(path)) return true; @@ -236,11 +236,11 @@ static void checkSelectorUse(DrvNames & selectors) namespace { -std::set searchByPrefix(const DrvInfos & allElems, std::string_view prefix) { +std::set searchByPrefix(const PackageInfos & allElems, std::string_view prefix) { constexpr std::size_t maxResults = 3; std::set result; - for (const auto & drvInfo : allElems) { - const auto drvName = DrvName { drvInfo.queryName() }; + for (const auto & packageInfo : allElems) { + const auto drvName = DrvName { packageInfo.queryName() }; if (hasPrefix(drvName.name, prefix)) { result.emplace(drvName.name); @@ -254,11 +254,11 @@ std::set searchByPrefix(const DrvInfos & allElems, std::string_view struct Match { - DrvInfo drvInfo; + PackageInfo packageInfo; std::size_t index; - Match(DrvInfo drvInfo_, std::size_t index_) - : drvInfo{std::move(drvInfo_)} + Match(PackageInfo packageInfo_, std::size_t index_) + : packageInfo{std::move(packageInfo_)} , index{index_} {} }; @@ -276,7 +276,7 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) StringSet multiple; for (auto & match : matches) { - auto & oneDrv = match.drvInfo; + auto & oneDrv = match.packageInfo; const auto drvName = DrvName { oneDrv.queryName() }; long comparison = 1; @@ -284,7 +284,7 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) const auto itOther = newest.find(drvName.name); if (itOther != newest.end()) { - auto & newestDrv = itOther->second.drvInfo; + auto & newestDrv = itOther->second.packageInfo; comparison = oneDrv.querySystem() == newestDrv.querySystem() ? 0 : @@ -319,23 +319,23 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) } // end namespace -static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, +static PackageInfos filterBySelector(EvalState & state, const PackageInfos & allElems, const Strings & args, bool newestOnly) { DrvNames selectors = drvNamesFromArgs(args); if (selectors.empty()) selectors.emplace_back("*"); - DrvInfos elems; + PackageInfos elems; std::set done; for (auto & selector : selectors) { std::vector matches; - for (const auto & [index, drvInfo] : enumerate(allElems)) { - const auto drvName = DrvName { drvInfo.queryName() }; + for (const auto & [index, packageInfo] : enumerate(allElems)) { + const auto drvName = DrvName { packageInfo.queryName() }; if (selector.matches(drvName)) { ++selector.hits; - matches.emplace_back(drvInfo, index); + matches.emplace_back(packageInfo, index); } } @@ -347,7 +347,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, haven't inserted before. */ for (auto & match : matches) if (done.insert(match.index).second) - elems.push_back(match.drvInfo); + elems.push_back(match.packageInfo); if (selector.hits == 0 && selector.fullName != "*") { const auto prefixHits = searchByPrefix(allElems, selector.name); @@ -376,7 +376,7 @@ static bool isPath(std::string_view s) static void queryInstSources(EvalState & state, InstallSourceInfo & instSource, const Strings & args, - DrvInfos & elems, bool newestOnly) + PackageInfos & elems, bool newestOnly) { InstallSourceType type = instSource.type; if (type == srcUnknown && args.size() > 0 && isPath(args.front())) @@ -392,7 +392,7 @@ static void queryInstSources(EvalState & state, /* Load the derivations from the (default or specified) Nix expression. */ - DrvInfos allElems; + PackageInfos allElems; loadDerivations(state, *instSource.nixExprPath, instSource.systemFilter, *instSource.autoArgs, "", allElems); @@ -413,7 +413,7 @@ static void queryInstSources(EvalState & state, loadSourceExpr(state, *instSource.nixExprPath, vArg); for (auto & i : args) { - Expr * eFun = state.parseExprFromString(i, state.rootPath(CanonPath::fromCwd())); + Expr * eFun = state.parseExprFromString(i, state.rootPath(".")); Value vFun, vTmp; state.eval(eFun, vFun); vTmp.mkApp(&vFun, &vArg); @@ -433,7 +433,7 @@ static void queryInstSources(EvalState & state, std::string name(path.name()); - DrvInfo elem(state, "", nullptr); + PackageInfo elem(state, "", nullptr); elem.setName(name); if (path.isDerivation()) { @@ -476,7 +476,7 @@ static void queryInstSources(EvalState & state, } -static void printMissing(EvalState & state, DrvInfos & elems) +static void printMissing(EvalState & state, PackageInfos & elems) { std::vector targets; for (auto & i : elems) @@ -494,7 +494,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) } -static bool keep(DrvInfo & drv) +static bool keep(PackageInfo & drv) { return drv.queryMetaBool("keep", false); } @@ -506,7 +506,7 @@ static void installDerivations(Globals & globals, debug("installing derivations"); /* Get the set of user environment elements to be installed. */ - DrvInfos newElems, newElemsTmp; + PackageInfos newElems, newElemsTmp; queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); /* If --prebuilt-only is given, filter out source-only packages. */ @@ -529,12 +529,12 @@ static void installDerivations(Globals & globals, while (true) { auto lockToken = optimisticLockProfile(profile); - DrvInfos allElems(newElems); + PackageInfos allElems(newElems); /* Add in the already installed derivations, unless they have the same name as a to-be-installed element. */ if (!globals.removeAll) { - DrvInfos installedElems = queryInstalled(*globals.state, profile); + PackageInfos installedElems = queryInstalled(*globals.state, profile); for (auto & i : installedElems) { DrvName drvName(i.queryName()); @@ -592,14 +592,14 @@ static void upgradeDerivations(Globals & globals, while (true) { auto lockToken = optimisticLockProfile(globals.profile); - DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + PackageInfos installedElems = queryInstalled(*globals.state, globals.profile); /* Fetch all derivations from the input file. */ - DrvInfos availElems; + PackageInfos availElems; queryInstSources(*globals.state, globals.instSource, args, availElems, false); /* Go through all installed derivations. */ - DrvInfos newElems; + PackageInfos newElems; for (auto & i : installedElems) { DrvName drvName(i.queryName()); @@ -617,7 +617,7 @@ static void upgradeDerivations(Globals & globals, priority. If there are still multiple matches, take the one with the highest version. Do not upgrade if it would decrease the priority. */ - DrvInfos::iterator bestElem = availElems.end(); + PackageInfos::iterator bestElem = availElems.end(); std::string bestVersion; for (auto j = availElems.begin(); j != availElems.end(); ++j) { if (comparePriorities(*globals.state, i, *j) > 0) @@ -687,7 +687,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) } -static void setMetaFlag(EvalState & state, DrvInfo & drv, +static void setMetaFlag(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & value) { auto v = state.allocValue(); @@ -711,7 +711,7 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) while (true) { std::string lockToken = optimisticLockProfile(globals.profile); - DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + PackageInfos installedElems = queryInstalled(*globals.state, globals.profile); /* Update all matching derivations. */ for (auto & i : installedElems) { @@ -745,13 +745,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", arg); } - DrvInfos elems; + PackageInfos elems; queryInstSources(*globals.state, globals.instSource, opArgs, elems, true); if (elems.size() != 1) throw Error("--set requires exactly one derivation"); - DrvInfo & drv(elems.front()); + PackageInfo & drv(elems.front()); if (globals.forceName != "") drv.setName(globals.forceName); @@ -786,10 +786,10 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, while (true) { auto lockToken = optimisticLockProfile(profile); - DrvInfos workingElems = queryInstalled(*globals.state, profile); + PackageInfos workingElems = queryInstalled(*globals.state, profile); for (auto & selector : selectors) { - DrvInfos::iterator split = workingElems.begin(); + PackageInfos::iterator split = workingElems.begin(); if (isPath(selector)) { StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector); split = std::partition( @@ -838,7 +838,7 @@ static bool cmpChars(char a, char b) } -static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) +static bool cmpElemByName(const PackageInfo & a, const PackageInfo & b) { auto a_name = a.queryName(); auto b_name = b.queryName(); @@ -891,7 +891,7 @@ void printTable(Table & table) typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; static VersionDiff compareVersionAgainstSet( - const DrvInfo & elem, const DrvInfos & elems, std::string & version) + const PackageInfo & elem, const PackageInfos & elems, std::string & version) { DrvName name(elem.queryName()); @@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet( } -static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printDrvPath, bool printMeta) +static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printDrvPath, bool printMeta) { using nlohmann::json; json topObj = json::object(); @@ -942,7 +942,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin }; { - DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); + PackageInfo::Outputs outputs = i.queryOutputs(printOutPath); json &outputObj = pkgObj["outputs"]; outputObj = json::object(); for (auto & j : outputs) { @@ -1032,7 +1032,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError("--attr-path(-P) only works with --available"); /* Obtain derivation information from the specified source. */ - DrvInfos availElems, installedElems; + PackageInfos availElems, installedElems; if (source == sInstalled || compareVersions || printStatus) installedElems = queryInstalled(*globals.state, globals.profile); @@ -1042,16 +1042,16 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) globals.instSource.systemFilter, *globals.instSource.autoArgs, attrPath, availElems); - DrvInfos elems_ = filterBySelector(*globals.state, + PackageInfos elems_ = filterBySelector(*globals.state, source == sInstalled ? installedElems : availElems, opArgs, false); - DrvInfos & otherElems(source == sInstalled ? availElems : installedElems); + PackageInfos & otherElems(source == sInstalled ? availElems : installedElems); /* Sort them by name. */ /* !!! */ - std::vector elems; + std::vector elems; for (auto & i : elems_) elems.push_back(i); sort(elems.begin(), elems.end(), cmpElemByName); @@ -1192,7 +1192,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs["outputName"] = i.queryOutputName(); if (printOutPath && !xmlOutput) { - DrvInfo::Outputs outputs = i.queryOutputs(); + PackageInfo::Outputs outputs = i.queryOutputs(); std::string s; for (auto & j : outputs) { if (!s.empty()) s += ';'; @@ -1212,7 +1212,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (xmlOutput) { XMLOpenElement item(xml, "item", attrs); - DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); + PackageInfo::Outputs outputs = i.queryOutputs(printOutPath); for (auto & j : outputs) { XMLAttrs attrs2; attrs2["name"] = j.first; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 34f6bd0055a..2f9c988d5f2 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -8,14 +8,16 @@ #include "eval.hh" #include "eval-inline.hh" #include "profiles.hh" +#include "print-ambiguous.hh" +#include namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv) +PackageInfos queryInstalled(EvalState & state, const Path & userEnv) { - DrvInfos elems; + PackageInfos elems; if (pathExists(userEnv + "/manifest.json")) throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); auto manifestFile = userEnv + "/manifest.nix"; @@ -29,7 +31,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) } -bool createUserEnv(EvalState & state, DrvInfos & elems, +bool createUserEnv(EvalState & state, PackageInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken) { @@ -55,7 +57,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ std::optional drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt; - DrvInfo::Outputs outputs = i.queryOutputs(true, true); + PackageInfo::Outputs outputs = i.queryOutputs(true, true); StringSet metaNames = i.queryMetaNames(); auto attrs = state.buildBindings(7 + outputs.size()); @@ -104,10 +106,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ - std::ostringstream str; - manifest.print(state.symbols, str, true); - auto manifestFile = state.store->addTextToStore("env-manifest.nix", - str.str(), references); + auto manifestFile = ({ + std::ostringstream str; + printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); + // TODO with C++20 we can use str.view() instead and avoid copy. + std::string str2 = str.str(); + StringSource source { str2 }; + state.store->addToStoreFromDump( + source, "env-manifest.nix", TextIngestionMethod {}, HashAlgorithm::SHA256, references); + }); /* Get the environment builder expression. */ Value envBuilder; @@ -128,7 +135,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Evaluate it. */ debug("evaluating user environment builder"); - state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); + state.forceValue(topLevel, topLevel.determinePos(noPos)); NixStringContext context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index af45d2d85d3..15da3fcb3f0 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -5,9 +5,9 @@ namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv); +PackageInfos queryInstalled(EvalState & state, const Path & userEnv); -bool createUserEnv(EvalState & state, DrvInfos & elems, +bool createUserEnv(EvalState & state, PackageInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 86b9be17d4b..86e6f008db7 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -1,9 +1,11 @@ #include "globals.hh" +#include "print-ambiguous.hh" #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" #include "attr-path.hh" +#include "signals.hh" #include "value-to-xml.hh" #include "value-to-json.hh" #include "store-api.hh" @@ -24,7 +26,6 @@ static int rootNr = 0; enum OutputKind { okPlain, okXML, okJSON }; - void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, bool evalOnly, OutputKind output, bool location, Expr * e) @@ -40,7 +41,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, for (auto & i : attrPaths) { Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); - state.forceValue(v, [&]() { return v.determinePos(noPos); }); + state.forceValue(v, v.determinePos(noPos)); NixStringContext context; if (evalOnly) { @@ -56,11 +57,12 @@ void processExpr(EvalState & state, const Strings & attrPaths, std::cout << std::endl; } else { if (strict) state.forceValueDeep(vRes); - vRes.print(state.symbols, std::cout); + std::set seen; + printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits::max()); std::cout << std::endl; } } else { - DrvInfos drvs; + PackageInfos drvs; getDerivations(state, v, "", autoArgs, drvs, false); for (auto & i : drvs) { auto drvPath = i.requireDrvPath(); @@ -166,7 +168,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { auto p = state->findFile(i); if (auto fn = p.getPhysicalPath()) - std::cout << fn->abs() << std::endl; + std::cout << fn->native() << std::endl; else throw Error("'%s' has no physical path", p); } @@ -182,7 +184,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs - ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) + ? state->parseExprFromString(i, state->rootPath(".")) : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index d361dc0ac26..99dbfe6e333 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -13,6 +13,7 @@ #include "shared.hh" #include "graphml.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include "path-with-outputs.hh" #include "posix-fs-canonicalise.hh" @@ -175,8 +176,11 @@ static void opAdd(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); + for (auto & i : opArgs) { + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + cout << fmt("%s\n", store->printStorePath(store->addToStore( + std::string(baseNameOf(i)), accessor, canonPath))); + } } @@ -196,8 +200,15 @@ static void opAddFixed(Strings opFlags, Strings opArgs) HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); - for (auto & i : opArgs) - std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); + for (auto & i : opArgs) { + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( + baseNameOf(i), + accessor, + canonPath, + method, + hashAlgo).path)); + } } @@ -541,7 +552,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path)); + HashResult hash = hashPath( + *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }, + + FileIngestionMethod::Recursive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } @@ -814,11 +828,9 @@ static void opServe(Strings opFlags, Strings opArgs) FdSink out(STDOUT_FILENO); /* Exchange the greeting. */ - unsigned int magic = readInt(in); - if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; - out.flush(); - ServeProto::Version clientVersion = readInt(in); + ServeProto::Version clientVersion = + ServeProto::BasicServerConnection::handshake( + out, in, SERVE_PROTOCOL_VERSION); ServeProto::ReadConn rconn { .from = in, @@ -938,8 +950,8 @@ static void opServe(Strings opFlags, Strings opArgs) store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { - assert(e.status); - out << e.status << e.msg(); + assert(e.info().status); + out << e.info().status << e.msg(); } break; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 02de796b5c7..9ea37ab4c40 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,25 +2,16 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "posix-source-accessor.hh" using namespace nix; -static FileIngestionMethod parseIngestionMethod(std::string_view input) -{ - if (input == "flat") { - return FileIngestionMethod::Flat; - } else if (input == "nar") { - return FileIngestionMethod::Recursive; - } else { - throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); - } -} - struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = FileIngestionMethod::Recursive; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() { @@ -37,7 +28,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "mode", - .shortName = 'n', .description = R"( How to compute the hash of the input. One of: @@ -48,45 +38,26 @@ struct CmdAddToStore : MixDryRun, StoreCommand )", .labels = {"hash-mode"}, .handler = {[this](std::string s) { - this->ingestionMethod = parseIngestionMethod(s); + this->caMethod = parseFileIngestionMethod(s); }}, }); + + addFlag(Flag::mkHashAlgoFlag(&hashAlgo)); } void run(ref store) override { if (!namePart) namePart = baseNameOf(path); - StringSink sink; - dumpPath(path, sink); - - auto narHash = hashString(HashAlgorithm::SHA256, sink.s); - - Hash hash = narHash; - if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(HashAlgorithm::SHA256); - readFile(path, hsink); - hash = hsink.finish().first; - } - - ValidPathInfo info { - *store, - std::move(*namePart), - FixedOutputInfo { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - .references = {}, - }, - narHash, - }; - info.narSize = sink.s.size(); - - if (!dryRun) { - auto source = StringSource(sink.s); - store->addToStore(info, source); - } - - logger->cout("%s", store->printStorePath(info.path)); + auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path); + + auto storePath = dryRun + ? store->computeStorePath( + *namePart, accessor, path2, caMethod, hashAlgo, {}).first + : store->addToStoreSlow( + *namePart, accessor, path2, caMethod, hashAlgo, {}).path; + + logger->cout("%s", store->printStorePath(storePath)); } }; @@ -110,7 +81,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - ingestionMethod = FileIngestionMethod::Flat; + caMethod = FileIngestionMethod::Flat; } std::string description() override diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 410feca2f2e..8d4717e150b 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -107,7 +107,7 @@ struct CmdConfigCheck : StoreCommand if (profileDir.find("/profiles/") == std::string::npos) dirs.insert(dir); } - } catch (SysError &) {} + } catch (SystemError &) {} } if (!dirs.empty()) { diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 4dada8e0e61..8afcbe98282 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -377,7 +377,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? - ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); + ei.msg = HintFmt("error processing connection: %1%", ei.msg.str()); logError(ei); } } diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index f116681ab3a..331cbdd8895 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -9,10 +9,11 @@ Store derivations are used internally by Nix. They are store paths with extension `.drv` that represent the build-time dependency graph to which a Nix expression evaluates. -[store derivation]: ../../glossary.md#gloss-store-derivation -The JSON format is documented under the [`derivation show`] command. +[store derivation]: @docroot@/glossary.md#gloss-store-derivation -[`derivation show`]: ./nix3-derivation-show.md +`nix derivation add` takes a single derivation in the following format: + +{{#include ../../protocols/json/derivation.md}} )"" diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 1296e2885f4..2437ea08f41 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -5,8 +5,6 @@ R""( * Show the [store derivation] that results from evaluating the Hello package: - [store derivation]: ../../glossary.md#gloss-store-derivation - ```console # nix derivation show nixpkgs#hello { @@ -48,62 +46,12 @@ a Nix expression evaluates. By default, this command only shows top-level derivations, but with `--recursive`, it also shows their dependencies. -The JSON output is a JSON object whose keys are the store paths of the -derivations, and whose values are a JSON object with the following -fields: - -* `name`: The name of the derivation. This is used when calculating the - store paths of the derivation's outputs. - -* `outputs`: Information about the output paths of the - derivation. This is a JSON object with one member per output, where - the key is the output name and the value is a JSON object with these - fields: - - * `path`: The output path. - * `hashAlgo`: For fixed-output derivations, the hashing algorithm - (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a - NAR hash rather than a flat file hash. - * `hash`: For fixed-output derivations, the expected content hash in - base-16. - - Example: - - ```json - "outputs": { - "out": { - "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - "hashAlgo": "r:sha256", - "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - } - } - ``` - -* `inputSrcs`: A list of store paths on which this derivation depends. - -* `inputDrvs`: A JSON object specifying the derivations on which this - derivation depends, and what outputs of those derivations. For - example, - - ```json - "inputDrvs": { - "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - } - ``` - - specifies that this derivation depends on the `dev` output of - `curl`, and the `out` output of `unzip`. - -* `system`: The system type on which this derivation is to be built - (e.g. `x86_64-linux`). +[store derivation]: @docroot@/glossary.md#gloss-store-derivation -* `builder`: The absolute path of the program to be executed to run - the build. Typically this is the `bash` shell - (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). +`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format: -* `args`: The command-line arguments passed to the `builder`. +[store path]: @docroot@/glossary.md#gloss-store-path -* `env`: The environment passed to the `builder`. +{{#include ../../protocols/json/derivation.md}} )"" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c1153624eb3..482c1e45b36 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -223,7 +223,11 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = ({ + StringSource source { getEnvSh }; + evalStore->addToStoreFromDump( + source, "get-env.sh", TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + }); drv.args = {store->printStorePath(getEnvShPath)}; @@ -293,7 +297,6 @@ struct Common : InstallableCommand, MixProfile "NIX_LOG_FD", "NIX_REMOTE", "PPID", - "SHELL", "SHELLOPTS", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", @@ -351,7 +354,7 @@ struct Common : InstallableCommand, MixProfile for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); - out << "eval \"$shellHook\"\n"; + out << "eval \"${shellHook:-}\"\n"; auto script = out.str(); @@ -372,7 +375,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto builtPaths = Installable::toStorePaths( + auto builtPaths = Installable::toStorePathSet( getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); for (auto & path: builtPaths) { auto from = store->printStorePath(path); @@ -600,7 +603,7 @@ struct CmdDevelop : Common, MixEnvironment setEnviron(); // prevent garbage collection until shell exits - setenv("NIX_GCROOT", gcroot.data(), 1); + setenv("NIX_GCROOT", gcroot.c_str(), 1); Path shell = "bash"; @@ -627,7 +630,7 @@ struct CmdDevelop : Common, MixEnvironment bool found = false; - for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { + for (auto & path : Installable::toStorePathSet(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { auto s = store->printStorePath(path) + "/bin/bash"; if (pathExists(s)) { shell = s; @@ -643,6 +646,10 @@ struct CmdDevelop : Common, MixEnvironment ignoreException(); } + // Override SHELL with the one chosen for this environment. + // This is to make sure the system shell doesn't leak into the build environment. + setenv("SHELL", shell.c_str(), 1); + // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} diff --git a/src/nix/eval.cc b/src/nix/eval.cc index b34af34e0eb..2044c8c2b40 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -66,7 +66,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (apply) { auto vApply = state->allocValue(); - state->eval(state->parseExprFromString(*apply, state->rootPath(CanonPath::fromCwd())), *vApply); + state->eval(state->parseExprFromString(*apply, state->rootPath(".")), *vApply); auto vRes = state->allocValue(); state->callFunction(*vApply, *v, *vRes, noPos); v = vRes; @@ -98,13 +98,13 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } catch (Error & e) { e.addTrace( state->positions[attr.pos], - hintfmt("while evaluating the attribute '%s'", name)); + HintFmt("while evaluating the attribute '%s'", name)); throw; } } } else - throw TypeError("value at '%s' is not a string or an attribute set", state->positions[pos]); + state->error("value at '%s' is not a string or an attribute set", state->positions[pos]).debugThrow(); }; recurse(*v, pos, *writeTo); @@ -121,7 +121,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else { state->forceValueDeep(*v); - logger->cout("%s", printValue(*state, *v)); + logger->cout("%s", ValuePrinter(*state, *v, PrintOptions { .force = true })); } } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 77abde5d8fb..4b013c67c5f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -391,13 +391,23 @@ struct CmdFlakeCheck : FlakeCommand auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional { try { - auto drvInfo = getDerivation(*state, v, false); - if (!drvInfo) + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking derivation %s", attrPath)); + auto packageInfo = getDerivation(*state, v, false); + if (!packageInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); - // FIXME: check meta attributes - return drvInfo->queryDrvPath(); + else { + // FIXME: check meta attributes + auto storePath = packageInfo->queryDrvPath(); + if (storePath) { + logger->log(lvlInfo, + fmt("derivation evaluated to %s", + store->printStorePath(storePath.value()))); + } + return storePath; + } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the derivation '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the derivation '%s'", attrPath)); reportError(e); } return std::nullopt; @@ -416,13 +426,15 @@ struct CmdFlakeCheck : FlakeCommand } #endif } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the app definition '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath)); reportError(e); } }; auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking overlay '%s'", attrPath)); state->forceValue(v, pos); if (!v.isLambda()) { throw Error("overlay is not a function, but %s instead", showType(v)); @@ -438,16 +450,18 @@ struct CmdFlakeCheck : FlakeCommand // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the overlay '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the overlay '%s'", attrPath)); reportError(e); } }; auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking NixOS module '%s'", attrPath)); state->forceValue(v, pos); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the NixOS module '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the NixOS module '%s'", attrPath)); reportError(e); } }; @@ -465,7 +479,7 @@ struct CmdFlakeCheck : FlakeCommand state->forceAttrs(*attr.value, attr.pos, ""); auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); if (state->isDerivation(*attr.value)) { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath2)); checkDerivation(attrPath2, *attr.value, attr.pos); } else @@ -473,14 +487,14 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the Hydra jobset '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the Hydra jobset '%s'", attrPath)); reportError(e); } }; auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS configuration '%s'", attrPath)); Bindings & bindings(*state->allocBindings(0)); auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; @@ -488,14 +502,14 @@ struct CmdFlakeCheck : FlakeCommand if (!state->isDerivation(*vToplevel)) throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the NixOS configuration '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the NixOS configuration '%s'", attrPath)); reportError(e); } }; auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking template '%s'", attrPath)); state->forceAttrs(v, pos, ""); @@ -522,19 +536,21 @@ struct CmdFlakeCheck : FlakeCommand throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); reportError(e); } }; auto checkBundler = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking bundler '%s'", attrPath)); state->forceValue(v, pos); if (!v.isLambda()) throw Error("bundler must be a function"); // TODO: check types of inputs/outputs? } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); reportError(e); } }; @@ -548,7 +564,7 @@ struct CmdFlakeCheck : FlakeCommand enumerateOutputs(*state, *vFlake, [&](const std::string & name, Value & vOutput, const PosIdx pos) { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); try { @@ -754,14 +770,15 @@ struct CmdFlakeCheck : FlakeCommand warn("unknown flake output '%s'", name); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking flake output '%s'", name)); + e.addTrace(resolve(pos), HintFmt("while checking flake output '%s'", name)); reportError(e); } }); } if (build && !drvPaths.empty()) { - Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + Activity act(*logger, lvlInfo, actUnknown, + fmt("running %d flake checks", drvPaths.size())); store->buildPaths(drvPaths); } if (hasErrors) @@ -836,8 +853,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand createDirs(to.abs()); for (auto & [name, entry] : from.readDirectory()) { - auto from2 = from + name; - auto to2 = to + name; + auto from2 = from / name; + auto to2 = to / name; auto st = from2.lstat(); if (st.type == InputAccessor::tDirectory) copyDir(from2, to2); @@ -1292,7 +1309,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { auto aType = visitor.maybeGetAttr("type"); if (!aType || aType->getString() != "app") - throw EvalError("not an app definition"); + state->error("not an app definition").debugThrow(); if (json) { j.emplace("type", "app"); } else { diff --git a/src/nix/hash-convert.md b/src/nix/hash-convert.md new file mode 100644 index 00000000000..dfb2154436e --- /dev/null +++ b/src/nix/hash-convert.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Convert a hash to `nix32` (a base-32 encoding with a Nix-specific character set). + + ```console + $ nix hash convert --hash-algo sha1 --to nix32 800d59cfcd3c05e900cb4e214be48f6b886a08df + vw46m23bizj4n8afrc0fj19wrp7mj3c0 + ``` + +* Convert a hash to [the `sri` format](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) that includes an algorithm specification: + + ```console + # nix hash convert --hash-algo sha1 800d59cfcd3c05e900cb4e214be48f6b886a08df + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + + or with an explicit `--to` format: + + ```console + # nix hash convert --hash-algo sha1 --to sri 800d59cfcd3c05e900cb4e214be48f6b886a08df + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + +* Assert the input format of the hash: + + ```console + # nix hash convert --hash-algo sha256 --from nix32 ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32' + + # nix hash convert --hash-algo sha256 --from nix32 1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s + sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + ``` + +# Description + +`nix hash convert` converts hashes from one encoding to another. + +)"" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0bba3b7d25e..eec1c0eae7b 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,6 +5,7 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "posix-source-accessor.hh" using namespace nix; @@ -88,14 +89,8 @@ struct CmdHashBase : Command else hashSink = std::make_unique(ha); - switch (mode) { - case FileIngestionMethod::Flat: - readFile(path, *hashSink); - break; - case FileIngestionMethod::Recursive: - dumpPath(path, *hashSink); - break; - } + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path); + dumpPath(accessor, canonPath, *hashSink, mode); Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); @@ -146,7 +141,7 @@ struct CmdHashConvert : Command CmdHashConvert(): to(HashFormat::SRI) { addFlag(Args::Flag::mkHashFormatOptFlag("from", &from)); addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to)); - addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo)); + addFlag(Args::Flag::mkHashAlgoOptFlag(&algo)); expectArgs({ .label = "hashes", .handler = {&hashStrings}, @@ -155,15 +150,14 @@ struct CmdHashConvert : Command std::string description() override { - std::string descr( "convert between different hash formats. Choose from: "); - auto iter = hashFormats.begin(); - assert(iter != hashFormats.end()); - descr += *iter++; - while (iter != hashFormats.end()) { - descr += ", " + *iter++; - } + return "convert between hash formats"; + } - return descr; + std::string doc() override + { + return + #include "hash-convert.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/local.mk b/src/nix/local.mk index a21aa705ffb..1d6f560d681 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -18,7 +18,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) +nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 231456c9c6c..63f97f2d3b6 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -72,7 +72,7 @@ struct MixLs : virtual Args, MixJSON if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); for (auto & [name, type] : names) - showFile(curPath + name, relPath + "/" + name); + showFile(curPath / name, relPath + "/" + name); } else showFile(curPath, relPath); }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index bbfeb8aa4f7..6e3f878d974 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -9,6 +9,7 @@ #include "attr-path.hh" #include "eval-inline.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include @@ -122,7 +123,10 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash); + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(tmpFile); + auto info = store->addToStoreSlow( + *name, accessor, canonPath, + ingestionMethod, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -257,6 +261,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; + bool unpack = false; std::optional name; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::optional expectedHash; @@ -289,6 +294,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .handler = {&executable, true}, }); + addFlag({ + .longName = "unpack", + .description = + "Unpack the archive (which must be a tarball or zip file) and add " + "the result to the Nix store.", + .handler = {&unpack, true}, + }); + expectArg("url", &url); } @@ -305,7 +318,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, unpack, executable); if (json) { auto res = nlohmann::json::object(); diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index 5d7fcc0ecc7..9811b9ec920 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,13 +6,13 @@ R""( ```console # nix profile list - Index: 0 + Name: gdb Flake attribute: legacyPackages.x86_64-linux.gdb Original flake URL: flake:nixpkgs Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 - Index: 1 + Name: blender-bin Flake attribute: packages.x86_64-linux.default Original flake URL: flake:blender-bin Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender @@ -26,7 +26,7 @@ R""( # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default ``` - will build the package with index 1 shown above. + will build the package `blender-bin` shown above. # Description @@ -34,10 +34,14 @@ This command shows what packages are currently installed in a profile. For each installed package, it shows the following information: -* `Index`: An integer that can be used to unambiguously identify the +* `Name`: A unique name used to unambiguously identify the package in invocations of `nix profile remove` and `nix profile upgrade`. +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile upgrade`. + (*Deprecated, will be removed in a future version in favor of `Name`.*) + * `Flake attribute`: The flake output attribute path that provides the package (e.g. `packages.x86_64-linux.hello`). diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index ba85441d801..1f653225015 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -2,16 +2,10 @@ R""( # Examples -* Remove a package by position: +* Remove a package by name: ```console - # nix profile remove 3 - ``` - -* Remove a package by attribute path: - - ```console - # nix profile remove packages.x86_64-linux.hello + # nix profile remove hello ``` * Remove all packages: diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 39cca428b0a..432b8fa9431 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -9,19 +9,10 @@ R""( # nix profile upgrade '.*' ``` -* Upgrade a specific package: +* Upgrade a specific package by name: ```console - # nix profile upgrade packages.x86_64-linux.hello - ``` - -* Upgrade a specific profile element by number: - - ```console - # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … - - # nix profile upgrade 0 + # nix profile upgrade hello ``` # Description diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 317c3e7c5c4..44726881fa9 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -10,6 +10,8 @@ #include "../nix-env/user-env.hh" #include "profiles.hh" #include "names.hh" +#include "url.hh" +#include "flake/url-name.hh" #include #include @@ -79,11 +81,6 @@ struct ProfileElement return showVersions(versions); } - bool operator < (const ProfileElement & other) const - { - return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); - } - void updateStorePaths( ref evalStore, ref store, @@ -106,7 +103,9 @@ struct ProfileElement struct ProfileManifest { - std::vector elements; + using ProfileElementName = std::string; + + std::map elements; ProfileManifest() { } @@ -126,6 +125,7 @@ struct ProfileManifest sOriginalUrl = "originalUri"; break; case 2: + case 3: sUrl = "url"; sOriginalUrl = "originalUrl"; break; @@ -133,7 +133,9 @@ struct ProfileManifest throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); } - for (auto & e : json["elements"]) { + auto elems = json["elements"]; + for (auto & elem : elems.items()) { + auto & e = elem.value(); ProfileElement element; for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); @@ -149,7 +151,15 @@ struct ProfileManifest e["outputs"].get() }; } - elements.emplace_back(std::move(element)); + + std::string name = + elems.is_object() + ? elem.key() + : element.source + ? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier()) + : element.identifier(); + + addElement(name, std::move(element)); } } @@ -158,20 +168,39 @@ struct ProfileManifest state.allowPath(state.store->followLinksToStore(profile)); state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); - auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile)); - for (auto & drvInfo : drvInfos) { + for (auto & packageInfo : packageInfos) { ProfileElement element; - element.storePaths = {drvInfo.queryOutPath()}; - elements.emplace_back(std::move(element)); + element.storePaths = {packageInfo.queryOutPath()}; + addElement(std::move(element)); } } } + void addElement(std::string_view nameCandidate, ProfileElement element) + { + std::string finalName(nameCandidate); + for (int i = 1; elements.contains(finalName); ++i) + finalName = nameCandidate + "-" + std::to_string(i); + + elements.insert_or_assign(finalName, std::move(element)); + } + + void addElement(ProfileElement element) + { + auto name = + element.source + ? getNameFromURL(parseURL(element.source->to_string())) + : std::nullopt; + auto name2 = name ? *name : element.identifier(); + addElement(name2, std::move(element)); + } + nlohmann::json toJSON(Store & store) const { - auto array = nlohmann::json::array(); - for (auto & element : elements) { + auto es = nlohmann::json::object(); + for (auto & [name, element] : elements) { auto paths = nlohmann::json::array(); for (auto & path : element.storePaths) paths.push_back(store.printStorePath(path)); @@ -185,11 +214,11 @@ struct ProfileManifest obj["attrPath"] = element.source->attrPath; obj["outputs"] = element.source->outputs; } - array.push_back(obj); + es[name] = obj; } nlohmann::json json; - json["version"] = 2; - json["elements"] = array; + json["version"] = 3; + json["elements"] = es; return json; } @@ -200,7 +229,7 @@ struct ProfileManifest StorePathSet references; Packages pkgs; - for (auto & element : elements) { + for (auto & [name, element] : elements) { for (auto & path : element.storePaths) { if (element.active) pkgs.emplace_back(store->printStorePath(path), true, element.priority); @@ -242,33 +271,27 @@ struct ProfileManifest static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) { - auto prevElems = prev.elements; - std::sort(prevElems.begin(), prevElems.end()); - - auto curElems = cur.elements; - std::sort(curElems.begin(), curElems.end()); - - auto i = prevElems.begin(); - auto j = curElems.begin(); + auto i = prev.elements.begin(); + auto j = cur.elements.begin(); bool changes = false; - while (i != prevElems.end() || j != curElems.end()) { - if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) { - logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions()); + while (i != prev.elements.end() || j != cur.elements.end()) { + if (j != cur.elements.end() && (i == prev.elements.end() || i->first > j->first)) { + logger->cout("%s%s: ∅ -> %s", indent, j->second.identifier(), j->second.versions()); changes = true; ++j; } - else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) { - logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions()); + else if (i != prev.elements.end() && (j == cur.elements.end() || i->first < j->first)) { + logger->cout("%s%s: %s -> ∅", indent, i->second.identifier(), i->second.versions()); changes = true; ++i; } else { - auto v1 = i->versions(); - auto v2 = j->versions(); + auto v1 = i->second.versions(); + auto v2 = j->second.versions(); if (v1 != v2) { - logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2); + logger->cout("%s%s: %s -> %s", indent, i->second.identifier(), v1, v2); changes = true; } ++i; @@ -367,7 +390,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.updateStorePaths(getEvalStore(), store, res); - manifest.elements.push_back(std::move(element)); + manifest.addElement(std::move(element)); } try { @@ -377,7 +400,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 auto findRefByFilePath = [&](Iterator begin, Iterator end) { for (auto it = begin; it != end; it++) { - auto profileElement = *it; + auto & profileElement = it->second; for (auto & storePath : profileElement.storePaths) { if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { return std::pair(conflictError.fileA, profileElement.toInstallables(*store)); @@ -445,7 +468,7 @@ class MixProfileElementMatchers : virtual Args std::string pattern; std::regex reg; }; - typedef std::variant Matcher; + typedef std::variant Matcher; std::vector getMatchers(ref store) { @@ -453,7 +476,7 @@ class MixProfileElementMatchers : virtual Args for (auto & s : _matchers) { if (auto n = string2Int(s)) - res.push_back(*n); + throw Error("'nix profile' no longer supports indices ('%d')", *n); else if (store->isStorePath(s)) res.push_back(s); else @@ -463,16 +486,17 @@ class MixProfileElementMatchers : virtual Args return res; } - bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector & matchers) + bool matches( + const Store & store, + const std::string & name, + const ProfileElement & element, + const std::vector & matchers) { for (auto & matcher : matchers) { - if (auto n = std::get_if(&matcher)) { - if (*n == pos) return true; - } else if (auto path = std::get_if(&matcher)) { + if (auto path = std::get_if(&matcher)) { if (element.storePaths.count(store.parseStorePath(*path))) return true; } else if (auto regex = std::get_if(&matcher)) { - if (element.source - && std::regex_match(element.source->attrPath, regex->reg)) + if (std::regex_match(name, regex->reg)) return true; } } @@ -503,10 +527,9 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem ProfileManifest newManifest; - for (size_t i = 0; i < oldManifest.elements.size(); ++i) { - auto & element(oldManifest.elements[i]); - if (!matches(*store, element, i, matchers)) { - newManifest.elements.push_back(std::move(element)); + for (auto & [name, element] : oldManifest.elements) { + if (!matches(*store, name, element, matchers)) { + newManifest.elements.insert_or_assign(name, std::move(element)); } else { notice("removing '%s'", element.identifier()); } @@ -519,11 +542,9 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem if (removedCount == 0) { for (auto matcher: matchers) { - if (const size_t * index = std::get_if(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if(&matcher)){ + if (const Path * path = std::get_if(&matcher)) { warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)){ + } else if (const RegexPattern * regex = std::get_if(&matcher)) { warn("'%s' does not match any packages", regex->pattern); } } @@ -554,64 +575,84 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); Installables installables; - std::vector indices; + std::vector elems; + auto matchedCount = 0; auto upgradedCount = 0; - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - if (element.source - && !element.source->originalRef.input.isLocked() - && matches(*store, element, i, matchers)) - { - upgradedCount++; - - Activity act(*logger, lvlChatty, actUnknown, - fmt("checking '%s' for updates", element.source->attrPath)); - - auto installable = make_ref( - this, - getEvalState(), - FlakeRef(element.source->originalRef), - "", - element.source->outputs, - Strings{element.source->attrPath}, - Strings{}, - lockFlags); - - auto derivedPaths = installable->toDerivedPaths(); - if (derivedPaths.empty()) continue; - auto * infop = dynamic_cast(&*derivedPaths[0].info); - // `InstallableFlake` should use `ExtraPathInfoFlake`. - assert(infop); - auto & info = *infop; + for (auto & [name, element] : manifest.elements) { + if (!matches(*store, name, element, matchers)) { + continue; + } - if (info.flake.lockedRef.input.isLocked() && element.source->lockedRef == info.flake.lockedRef) continue; + matchedCount++; - printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); + if (!element.source) { + warn( + "Found package '%s', but it was not installed from a flake, so it can't be checked for upgrades!", + element.identifier() + ); + continue; + } + if (element.source->originalRef.input.isLocked()) { + warn( + "Found package '%s', but it was installed from a locked flake reference so it can't be upgraded!", + element.identifier() + ); + continue; + } - element.source = ProfileElementSource { - .originalRef = installable->flakeRef, - .lockedRef = info.flake.lockedRef, - .attrPath = info.value.attrPath, - .outputs = installable->extendedOutputsSpec, - }; + upgradedCount++; + + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + auto installable = make_ref( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + "", + element.source->outputs, + Strings{element.source->attrPath}, + Strings{}, + lockFlags); + + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto * infop = dynamic_cast(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; + + if (info.flake.lockedRef.input.isLocked() + && element.source->lockedRef == info.flake.lockedRef) + continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); + + element.source = ProfileElementSource { + .originalRef = installable->flakeRef, + .lockedRef = info.flake.lockedRef, + .attrPath = info.value.attrPath, + .outputs = installable->extendedOutputsSpec, + }; - installables.push_back(installable); - indices.push_back(i); - } + installables.push_back(installable); + elems.push_back(&element); } if (upgradedCount == 0) { - for (auto & matcher : matchers) { - if (const size_t * index = std::get_if(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if(&matcher)){ - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)){ - warn("'%s' does not match any packages", regex->pattern); + if (matchedCount == 0) { + for (auto & matcher : matchers) { + if (const Path * path = std::get_if(&matcher)) { + warn("'%s' does not match any paths", *path); + } else if (const RegexPattern * regex = std::get_if(&matcher)) { + warn("'%s' does not match any packages", regex->pattern); + } } + } else { + warn("Found some packages but none of them could be upgraded."); } warn ("Use 'nix profile list' to see the current profile."); } @@ -622,7 +663,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); - auto & element = manifest.elements[indices.at(i)]; + auto & element = *elems.at(i); element.updateStorePaths( getEvalStore(), store, @@ -654,11 +695,11 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro if (json) { std::cout << manifest.toJSON(*store).dump() << "\n"; } else { - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); + for (const auto & [i, e] : enumerate(manifest.elements)) { + auto & [name, element] = e; if (i) logger->cout(""); - logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", - i, + logger->cout("Name: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + name, element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); if (element.source) { logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); diff --git a/src/nix/run.cc b/src/nix/run.cc index efc0c56a178..e868376790f 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -114,7 +114,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment setEnviron(); - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); + std::vector pathAdditions; while (!todo.empty()) { auto path = todo.front(); @@ -122,16 +122,19 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (!done.insert(path).second) continue; if (true) - unixPath.push_front(store->printStorePath(path) + "/bin"); + pathAdditions.push_back(store->printStorePath(path) + "/bin"); - auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + auto propPath = CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"; if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } - setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); + auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); + unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = concatStringsSep(":", unixPath); + setenv("PATH", unixPathString.c_str(), 1); Strings args; for (auto & arg : command) args.push_back(arg); diff --git a/src/nix/search.cc b/src/nix/search.cc index ef0139e093f..97ef1375ed2 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -67,11 +67,9 @@ struct CmdSearch : InstallableValueCommand, MixJSON settings.readOnlyMode = true; evalSettings.enableImportFromDerivation.setDefault(false); - // Empty search string should match all packages - // Use "^" here instead of ".*" due to differences in resulting highlighting - // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + // Recommend "^" here instead of ".*" due to differences in resulting highlighting if (res.empty()) - res.push_back("^"); + throw UsageError("Must provide at least one regex! To match all packages, use '%s'.", "nix search ^"); std::vector regexes; std::vector excludeRegexes; diff --git a/src/nix/search.md b/src/nix/search.md index 0c5d22549aa..f65ac9b1748 100644 --- a/src/nix/search.md +++ b/src/nix/search.md @@ -5,7 +5,7 @@ R""( * Show all packages in the `nixpkgs` flake: ```console - # nix search nixpkgs + # nix search nixpkgs ^ * legacyPackages.x86_64-linux.AMB-plugins (0.8.1) A set of ambisonics ladspa plugins @@ -34,7 +34,7 @@ R""( * Show all packages in the flake in the current directory: ```console - # nix search + # nix search . ^ ``` * Search for Firefox or Chromium: @@ -64,11 +64,16 @@ R""( `nix search` searches [*installable*](./nix.md#installables) (which can be evaluated, that is, a flake or Nix expression, but not a store path or store derivation path) for packages whose name or description matches all of the -regular expressions *regex*. For each matching package, It prints the +regular expressions *regex*. For each matching package, It prints the full attribute name (from the root of the [installable](./nix.md#installables)), the version and the `meta.description` field, highlighting the substrings that -were matched by the regular expressions. If no regular expressions are -specified, all packages are shown. +were matched by the regular expressions. + +To show all packages, use the regular expression `^`. In contrast to `.*`, +it avoids highlighting the entire name and description of every package. + +> Note that in this context, `^` is the regex character to match the beginning of a string, *not* the delimiter for +> [selecting a derivation output](@docroot@/command-ref/new-cli/nix.md#derivation-output-selection). # Flake output attributes diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a57a407e6b3..dfef4486931 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -112,7 +112,7 @@ struct CmdSign : StorePathsCommand std::string description() override { - return "sign store paths"; + return "sign store paths with a local key"; } void run(ref store, StorePaths && storePaths) override @@ -121,6 +121,7 @@ struct CmdSign : StorePathsCommand throw UsageError("you must specify a secret key file using '-k'"); SecretKey secretKey(readFile(secretKeyFile)); + LocalSigner signer(std::move(secretKey)); size_t added{0}; @@ -129,7 +130,7 @@ struct CmdSign : StorePathsCommand auto info2(*info); info2.sigs.clear(); - info2.sign(*store, secretKey); + info2.sign(*store, signer); assert(!info2.sigs.empty()); if (!info->sigs.count(*info2.sigs.begin())) { diff --git a/src/nix/verify.cc b/src/nix/verify.cc index f0234f7befe..2a0cbd19ffb 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -5,6 +5,7 @@ #include "thread-pool.hh" #include "references.hh" #include "signals.hh" +#include "keys.hh" #include diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index aecf6592222..e299585ff88 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -225,7 +225,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & [name, type] : names) - visitPath(p + name); + visitPath(p / name); } else if (st->type == SourceAccessor::Type::tRegular) { diff --git a/tests/functional/add.sh b/tests/functional/add.sh index d0fedcb251c..762e01dbe37 100644 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -37,9 +37,11 @@ clearStore path3=$(nix store add-path ./dummy) [[ "$path1" == "$path2" ]] [[ "$path1" == "$path3" ]] + path4=$(nix store add --mode nar --hash-algo sha1 ./dummy) ) ( path1=$(nix store add --mode flat ./dummy) path2=$(nix store add-file ./dummy) [[ "$path1" == "$path2" ]] + path4=$(nix store add --mode flat --hash-algo sha1 ./dummy) ) diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 848988af996..8fef29f97a7 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -45,6 +45,7 @@ if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" fi coreutils=@coreutils@ +lsof=@lsof@ export dot=@dot@ export SHELL="@bash@" @@ -95,7 +96,7 @@ startDaemon() { fi # Start the daemon, wait for the socket to appear. rm -f $NIX_DAEMON_SOCKET_PATH - PATH=$DAEMON_PATH nix-daemon & + PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID for ((i = 0; i < 300; i++)); do @@ -148,7 +149,7 @@ fi isDaemonNewer () { [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) + local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] } diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh new file mode 100644 index 00000000000..63d88cbf35b --- /dev/null +++ b/tests/functional/debugger.sh @@ -0,0 +1,13 @@ +source common.sh + +clearStore + +# regression #9932 +echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' +echo ":env" | expect 1 nix eval --debugger --expr ' + let x.a = 1; in + with x; + (_: builtins.seq x.a (throw "oh snap")) x.a +' >debugger-test-out +grep -P 'with: .*a' debugger-test-out +grep -P 'static: .*x' debugger-test-out diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh index 9467feb8d23..3681bd09889 100644 --- a/tests/functional/dyn-drv/eval-outputOf.sh +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -14,7 +14,7 @@ nix --experimental-features 'nix-command' eval --impure --expr \ # resolve first. Adding a test so we don't liberalise it by accident. expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ 'builtins.outputOf (import ../dependencies.nix {}) "out"' \ - | grepQuiet "value is a set while a string was expected" + | grepQuiet "expected a string but found a set" # Test that "DrvDeep" string contexts are not supported at this time # diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index ec99fd95311..9937ecbce65 100644 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -40,3 +40,11 @@ if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then (! ls $NIX_STORE_DIR/*.drv) fi ls $eval_store/nix/store/*.drv + +clearStore +rm -rf "$eval_store" + +# Confirm that import-from-derivation builds on the build store +[[ $(nix eval --eval-store "$eval_store?require-sigs=false" --impure --raw --file ./ifd.nix) = hi ]] +ls $NIX_STORE_DIR/*dependencies-top/foobar +(! ls $eval_store/nix/store/*dependencies-top/foobar) diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index cb444f8df0d..09e606f7e0d 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -66,6 +66,9 @@ path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \" # In pure eval mode, fetchGit with a revision should succeed. [[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] +# But without a hash, it fails +expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "fetchGit requires a locked input" + # Fetch again. This should be cached. mv $repo ${repo}-tmp path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") @@ -205,6 +208,8 @@ path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; ur [[ $path3 = $path6 ]] [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] +expectStderr 1 nix eval --expr 'builtins.fetchTree { type = "git"; url = "file:///foo"; }' | grepQuiet "fetchTree requires a locked input" + # Explicit ref = "HEAD" should work, and produce the same outPath as without ref path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath") @@ -229,6 +234,18 @@ rev_tag2=$(git -C $repo rev-parse refs/tags/tag2) [[ $rev_tag2_nix = $rev_tag2 ]] unset _NIX_FORCE_HTTP +# Ensure .gitattributes is respected +touch $repo/not-exported-file +touch $repo/exported-wonky +echo "/not-exported-file export-ignore" >> $repo/.gitattributes +echo "/exported-wonky export-ignore=wonk" >> $repo/.gitattributes +git -C $repo add not-exported-file exported-wonky .gitattributes +git -C $repo commit -m 'Bla6' +rev5=$(git -C $repo rev-parse HEAD) +path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath") +[[ ! -e $path12/not-exported-file ]] +[[ -e $path12/exported-wonky ]] + # should fail if there is no repo rm -rf $repo/.git (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index 369cdc5db43..cd180815d69 100644 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -118,3 +118,55 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path git clone $rootRepo $cloneRepo pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") [[ $pathIndirect = $pathWithRelative ]] + +# Test submodule export-ignore interaction +git -C $rootRepo/sub config user.email "foobar@example.com" +git -C $rootRepo/sub config user.name "Foobar" + +echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes +# TBD possible semantics for submodules + exportIgnore +# echo "/sub/exclude-deep export-ignore" >> $rootRepo/.gitattributes +echo nope > $rootRepo/exclude-from-root +git -C $rootRepo add .gitattributes exclude-from-root +git -C $rootRepo commit -m "Add export-ignore" + +echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes +echo nope > $rootRepo/sub/exclude-from-sub +# TBD possible semantics for submodules + exportIgnore +# echo aye > $rootRepo/sub/exclude-from-root +git -C $rootRepo/sub add .gitattributes exclude-from-sub +git -C $rootRepo/sub commit -m "Add export-ignore (sub)" + +git -C $rootRepo add sub +git -C $rootRepo commit -m "Update submodule" + +git -C $rootRepo status + +# # TBD: not supported yet, because semantics are undecided and current implementation leaks rules from the root to submodules +# # exportIgnore can be used with submodules +# pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath") +# # find $pathWithExportIgnore +# # git -C $rootRepo archive --format=tar HEAD | tar -t +# # cp -a $rootRepo /tmp/rootRepo + +# [[ -e $pathWithExportIgnore/sub/content ]] +# [[ ! -e $pathWithExportIgnore/exclude-from-root ]] +# [[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]] +# TBD possible semantics for submodules + exportIgnore +# # root .gitattribute has no power across submodule boundary +# [[ -e $pathWithExportIgnore/sub/exclude-from-root ]] +# [[ -e $pathWithExportIgnore/sub/exclude-deep ]] + + +# exportIgnore can be explicitly disabled with submodules +pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath") +# find $pathWithoutExportIgnore + +[[ -e $pathWithoutExportIgnore/exclude-from-root ]] +[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] + +# exportIgnore defaults to false when submodules = true +pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; }).outPath") + +[[ -e $pathWithoutExportIgnore/exclude-from-root ]] +[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] diff --git a/tests/functional/fetchurl.sh b/tests/functional/fetchurl.sh index 8cd40c09f8d..a3620f52b6f 100644 --- a/tests/functional/fetchurl.sh +++ b/tests/functional/fetchurl.sh @@ -78,3 +78,9 @@ outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file: test -x $outPath/fetchurl.sh test -L $outPath/symlink + +# Make sure that *not* passing a outputHash fails. +requireDaemonNewerThan "2.20" +expected=100 +if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly +expectStderr $expected nix-build --expr '{ url }: builtins.derivation { name = "nix-cache-info"; system = "x86_64-linux"; builder = "builtin:fetchurl"; inherit url; outputHashMode = "flat"; }' --argstr url file://$narxz 2>&1 | grep 'must be a fixed-output or impure derivation' diff --git a/tests/functional/fixed.nix b/tests/functional/fixed.nix index babe71504cd..5bdf7933386 100644 --- a/tests/functional/fixed.nix +++ b/tests/functional/fixed.nix @@ -48,6 +48,15 @@ rec { (f ./fixed.builder1.sh "flat" "md5" "ddd8be4b179a529afa5f2ffae4b9858") ]; + badReferences = mkDerivation rec { + name = "bad-hash"; + builder = script; + script = builtins.toFile "installer.sh" "echo $script >$out"; + outputHash = "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik"; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + }; + # Test for building two derivations in parallel that produce the # same output path because they're fixed-output derivations. parallelSame = [ diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index f1e1ce420dc..d98d4cd15c5 100644 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -26,6 +26,11 @@ nix-build fixed.nix -A good2 --no-out-link echo 'testing reallyBad...' nix-instantiate fixed.nix -A reallyBad && fail "should fail" +if isDaemonNewer "2.20pre20240108"; then + echo 'testing fixed with references...' + expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths" +fi + # While we're at it, check attribute selection a bit more. echo 'testing attribute selection...' test $(nix-instantiate fixed.nix -A good.1 | wc -l) = 1 diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh new file mode 100644 index 00000000000..e1e53d36428 --- /dev/null +++ b/tests/functional/flakes/develop.sh @@ -0,0 +1,67 @@ +source ../common.sh + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local + +# Create flake under test. +cp ../shell-hello.nix ../config.nix $TEST_HOME/ +cat <$TEST_HOME/flake.nix +{ + inputs.nixpkgs.url = "$TEST_HOME/nixpkgs"; + outputs = {self, nixpkgs}: { + packages.$system.hello = (import ./config.nix).mkDerivation { + name = "hello"; + outputs = [ "out" "dev" ]; + meta.outputsToInstall = [ "out" ]; + buildCommand = ""; + }; + }; +} +EOF + +# Create fake nixpkgs flake. +mkdir -p $TEST_HOME/nixpkgs +cp ../config.nix ../shell.nix $TEST_HOME/nixpkgs +cat <$TEST_HOME/nixpkgs/flake.nix +{ + outputs = {self}: { + legacyPackages.$system.bashInteractive = (import ./shell.nix {}).bashInteractive; + }; +} +EOF + +cd $TEST_HOME + +# Test whether `nix develop` passes through environment variables. +[[ "$( + ENVVAR=a nix develop --no-write-lock-file .#hello < $fifo1) & +pid2=$! + +# Start a build. This should not be blocked by the GC in progress. outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E " with import ./config.nix; mkDerivation { name = \"non-blocking\"; - buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo\"; + buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo2\"; }") wait $pid +wait $pid2 (! test -e $running) (! test -e $dummy) diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh new file mode 100644 index 00000000000..9b3afc10b27 --- /dev/null +++ b/tests/functional/hash-convert.sh @@ -0,0 +1,105 @@ +source common.sh + +# Conversion with `nix hash` `nix-hash` and `nix hash convert` +try3() { + # $1 = hash algo + # $2 = expected hash in base16 + # $3 = expected hash in base32 + # $4 = expected hash in base64 + h64=$(nix hash convert --hash-algo "$1" --to base64 "$2") + [ "$h64" = "$4" ] + h64=$(nix-hash --type "$1" --to-base64 "$2") + [ "$h64" = "$4" ] + # Deprecated experiment + h64=$(nix hash to-base64 --type "$1" "$2") + [ "$h64" = "$4" ] + + sri=$(nix hash convert --hash-algo "$1" --to sri "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix-hash --type "$1" --to-sri "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash to-sri --type "$1" "$2") + [ "$sri" = "$1-$4" ] + h32=$(nix hash convert --hash-algo "$1" --to base32 "$2") + [ "$h32" = "$3" ] + h32=$(nix-hash --type "$1" --to-base32 "$2") + [ "$h32" = "$3" ] + h32=$(nix hash to-base32 --type "$1" "$2") + [ "$h32" = "$3" ] + h16=$(nix-hash --type "$1" --to-base16 "$h32") + [ "$h16" = "$2" ] + + h16=$(nix hash convert --hash-algo "$1" --to base16 "$h64") + [ "$h16" = "$2" ] + h16=$(nix hash to-base16 --type "$1" "$h64") + [ "$h16" = "$2" ] + h16=$(nix hash convert --to base16 "$sri") + [ "$h16" = "$2" ] + h16=$(nix hash to-base16 "$sri") + [ "$h16" = "$2" ] + + # + # Converting from SRI + # + + # Input hash algo auto-detected from SRI and output defaults to SRI as well. + sri=$(nix hash convert "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to base64 "$1-$4") + [ "$sri" = "$4" ] + + # + # Auto-detecting the input from algo and length. + # + + sri=$(nix hash convert --hash-algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --hash-algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format succeeds. + # + + sri=$(nix hash convert --hash-algo "$1" --from base16 "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" --from nix32 "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" --from base64 "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format fails. + # + + fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + +} + +try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" +try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" +try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/functional/hash.sh b/tests/functional/hash-path.sh similarity index 51% rename from tests/functional/hash.sh rename to tests/functional/hash-path.sh index 47eed51780f..6d096b29be9 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash-path.sh @@ -80,107 +80,3 @@ try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" rm $TEST_ROOT/hash-path/hello ln -s x $TEST_ROOT/hash-path/hello try2 md5 "f78b733a68f5edbdf9413899339eaa4a" - -# Conversion with `nix hash` `nix-hash` and `nix hash convert` -try3() { - # $1 = hash algo - # $2 = expected hash in base16 - # $3 = expected hash in base32 - # $4 = expected hash in base64 - h64=$(nix hash convert --algo "$1" --to base64 "$2") - [ "$h64" = "$4" ] - h64=$(nix-hash --type "$1" --to-base64 "$2") - [ "$h64" = "$4" ] - # Deprecated experiment - h64=$(nix hash to-base64 --type "$1" "$2") - [ "$h64" = "$4" ] - - sri=$(nix hash convert --algo "$1" --to sri "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix-hash --type "$1" --to-sri "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash to-sri --type "$1" "$2") - [ "$sri" = "$1-$4" ] - h32=$(nix hash convert --algo "$1" --to base32 "$2") - [ "$h32" = "$3" ] - h32=$(nix-hash --type "$1" --to-base32 "$2") - [ "$h32" = "$3" ] - h32=$(nix hash to-base32 --type "$1" "$2") - [ "$h32" = "$3" ] - h16=$(nix-hash --type "$1" --to-base16 "$h32") - [ "$h16" = "$2" ] - - h16=$(nix hash convert --algo "$1" --to base16 "$h64") - [ "$h16" = "$2" ] - h16=$(nix hash to-base16 --type "$1" "$h64") - [ "$h16" = "$2" ] - h16=$(nix hash convert --to base16 "$sri") - [ "$h16" = "$2" ] - h16=$(nix hash to-base16 "$sri") - [ "$h16" = "$2" ] - - # - # Converting from SRI - # - - # Input hash algo auto-detected from SRI and output defaults to SRI as well. - sri=$(nix hash convert "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --from sri "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --to sri "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --from sri --to sri "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --to base64 "$1-$4") - [ "$sri" = "$4" ] - - # - # Auto-detecting the input from algo and length. - # - - sri=$(nix hash convert --algo "$1" "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" "$3") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" "$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --algo "$1" "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" "$3") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" "$4") - [ "$sri" = "$1-$4" ] - - # - # Asserting input format succeeds. - # - - sri=$(nix hash convert --algo "$1" --from base16 "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" --from nix32 "$3") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" --from base64 "$4") - [ "$sri" = "$1-$4" ] - - # - # Asserting input format fails. - # - - fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - -} - -try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" -try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" -try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/functional/ifd.nix b/tests/functional/ifd.nix new file mode 100644 index 00000000000..d0b9b54add0 --- /dev/null +++ b/tests/functional/ifd.nix @@ -0,0 +1,10 @@ +with import ./config.nix; +import ( + mkDerivation { + name = "foo"; + bla = import ./dependencies.nix {}; + buildCommand = " + echo \\\"hi\\\" > $out + "; + } +) diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 39d053a0402..54ed6f5ddf2 100644 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -63,3 +63,7 @@ path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAdd path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out) [[ $(< $path6) = X ]] [[ $(< $TEST_ROOT/counter) = 5 ]] + +# Test nix/fetchurl.nix. +path7=$(nix build -L --no-link --print-out-paths --expr "import { impure = true; url = file://$PWD/impure-derivations.sh; }") +cmp $path7 $PWD/impure-derivations.sh diff --git a/tests/functional/impure-eval.sh b/tests/functional/impure-eval.sh new file mode 100644 index 00000000000..6c72f01d7af --- /dev/null +++ b/tests/functional/impure-eval.sh @@ -0,0 +1,35 @@ +source common.sh + +export REMOTE_STORE="dummy://" + +simpleTest () { + local expr=$1; shift + local result=$1; shift + # rest, extra args + + [[ "$(nix eval --impure --raw "$@" --expr "$expr")" == "$result" ]] +} + +# `builtins.storeDir` + +## Store dir follows `store` store setting +simpleTest 'builtins.storeDir' '/foo' --store "$REMOTE_STORE?store=/foo" +simpleTest 'builtins.storeDir' '/bar' --store "$REMOTE_STORE?store=/bar" + +# `builtins.currentSystem` + +## `system` alone affects by default +simpleTest 'builtins.currentSystem' 'foo' --system 'foo' +simpleTest 'builtins.currentSystem' 'bar' --system 'bar' + +## `system` affects if `eval-system` is an empty string +simpleTest 'builtins.currentSystem' 'foo' --system 'foo' --eval-system '' +simpleTest 'builtins.currentSystem' 'bar' --system 'bar' --eval-system '' + +## `eval-system` alone affects +simpleTest 'builtins.currentSystem' 'foo' --eval-system 'foo' +simpleTest 'builtins.currentSystem' 'bar' --eval-system 'bar' + +## `eval-system` overrides `system` +simpleTest 'builtins.currentSystem' 'bar' --system 'foo' --eval-system 'bar' +simpleTest 'builtins.currentSystem' 'baz' --system 'foo' --eval-system 'baz' diff --git a/tests/functional/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp index 345232d3f34..20e7b9e18e8 100644 --- a/tests/functional/lang/eval-fail-abort.err.exp +++ b/tests/functional/lang/eval-fail-abort.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'abort' builtin - at /pwd/lang/eval-fail-abort.nix:1:14: - 1| if true then abort "this should fail" else 1 | ^ 2| diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp index ad91a22aa5b..37e0bd9ee0a 100644 --- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'addDrvOutputDependencies' builtin - at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: - 1| builtins.addDrvOutputDependencies "" | ^ 2| diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp index bb389db4e4c..6828e03c8e7 100644 --- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'addDrvOutputDependencies' builtin - at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: - 17| 18| in builtins.addDrvOutputDependencies combo-path | ^ diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp index 07038111822..72b5e636897 100644 --- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'addDrvOutputDependencies' builtin - at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: - 8| 9| in builtins.addDrvOutputDependencies drv.outPath | ^ diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp index aeecd8167fc..0656ec81c1d 100644 --- a/tests/functional/lang/eval-fail-assert.err.exp +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -1,35 +1,27 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-assert.nix:4:3: - 3| 4| body = x "x"; | ^ 5| } … from call site - at /pwd/lang/eval-fail-assert.nix:4:10: - 3| 4| body = x "x"; | ^ 5| } … while calling 'x' - at /pwd/lang/eval-fail-assert.nix:2:7: - 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| error: assertion '(arg == "y")' failed - at /pwd/lang/eval-fail-assert.nix:2:12: - 1| let { 2| x = arg: assert arg == "y"; 123; | ^ diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp index 5f9a073dd81..6848a35ed80 100644 --- a/tests/functional/lang/eval-fail-attr-name-type.err.exp +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -1,20 +1,21 @@ error: … while evaluating the attribute 'puppy."${key}"' - at /pwd/lang/eval-fail-attr-name-type.nix:3:5: - 2| attrs = { 3| puppy.doggy = {}; | ^ 4| }; … while evaluating an attribute name - at /pwd/lang/eval-fail-attr-name-type.nix:7:17: - 6| in 7| attrs.puppy.${key} | ^ 8| - error: value is an integer while a string was expected + error: expected a string but found an integer: 1 + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + 6| in + 7| attrs.puppy.${key} + | ^ + 8| diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp index eb73e9a5298..5ae53034d18 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -1,10 +1,8 @@ error: … while evaluating a path segment - at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: - 1| "${x: x}" | ^ 2| - error: cannot coerce a function to a string + error: cannot coerce a function to a string: «lambda @ /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:4» diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp index ac14f329b78..170a3d13233 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -1,10 +1,8 @@ error: … while evaluating a path segment - at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: - 1| ''${x: x}'' | ^ 2| - error: cannot coerce a function to a string + error: cannot coerce a function to a string: «lambda @ /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:5» diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp index 07843a48017..6f907106b3f 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -1,11 +1,9 @@ error: … while evaluating a path segment - at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3: - 8| # The error message should not be too long. 9| ''${pkgs}'' | ^ 10| - error: cannot coerce a set to a string + error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «4294967295 attributes elided» }; «4294967294 attributes elided» }; «4294967293 attributes elided» } diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp index f0618d8ac7d..95e33a5fe45 100644 --- a/tests/functional/lang/eval-fail-blackhole.err.exp +++ b/tests/functional/lang/eval-fail-blackhole.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-blackhole.nix:2:3: - 1| let { 2| body = x; | ^ 3| x = y; error: infinite recursion encountered - at /pwd/lang/eval-fail-blackhole.nix:3:7: - 2| body = x; 3| x = y; | ^ diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp index 19b407c4706..0c6f614e8a7 100644 --- a/tests/functional/lang/eval-fail-call-primop.err.exp +++ b/tests/functional/lang/eval-fail-call-primop.err.exp @@ -1,12 +1,10 @@ error: … while calling the 'length' builtin - at /pwd/lang/eval-fail-call-primop.nix:1:1: - 1| builtins.length 1 | ^ 2| … while evaluating the first argument passed to builtins.length - error: value is an integer while a list was expected + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp index 5e204ba73f6..11b62340d61 100644 --- a/tests/functional/lang/eval-fail-deepseq.err.exp +++ b/tests/functional/lang/eval-fail-deepseq.err.exp @@ -1,24 +1,18 @@ error: … while calling the 'deepSeq' builtin - at /pwd/lang/eval-fail-deepseq.nix:1:1: - 1| builtins.deepSeq { x = abort "foo"; } 456 | ^ 2| … while evaluating the attribute 'x' - at /pwd/lang/eval-fail-deepseq.nix:1:20: - 1| builtins.deepSeq { x = abort "foo"; } 456 | ^ 2| … while calling the 'abort' builtin - at /pwd/lang/eval-fail-deepseq.nix:1:24: - 1| builtins.deepSeq { x = abort "foo"; } 456 | ^ 2| diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp index c5fa67523b9..834f9c67bc4 100644 --- a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'set' - at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: - 1| { 2| set = { "${"" + "b"}" = 1; }; | ^ 3| set = { "${"b" + ""}" = 2; }; error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 - at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: - 2| set = { "${"" + "b"}" = 1; }; 3| set = { "${"b" + ""}" = 2; }; | ^ diff --git a/tests/functional/lang/eval-fail-duplicate-traces.err.exp b/tests/functional/lang/eval-fail-duplicate-traces.err.exp new file mode 100644 index 00000000000..32ad9b37650 --- /dev/null +++ b/tests/functional/lang/eval-fail-duplicate-traces.err.exp @@ -0,0 +1,44 @@ +error: + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:9:3: + 8| in + 9| throwAfter 2 + | ^ + 10| + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:6:10: + 5| if n > 0 + 6| then throwAfter (n - 1) + | ^ + 7| else throw "Uh oh!"; + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:6:10: + 5| if n > 0 + 6| then throwAfter (n - 1) + | ^ + 7| else throw "Uh oh!"; + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-duplicate-traces.nix b/tests/functional/lang/eval-fail-duplicate-traces.nix new file mode 100644 index 00000000000..17ce374ece7 --- /dev/null +++ b/tests/functional/lang/eval-fail-duplicate-traces.nix @@ -0,0 +1,9 @@ +# Check that we only omit duplicate stack traces when there's a bunch of them. +# Here, there's only a couple duplicate entries, so we output them all. +let + throwAfter = n: + if n > 0 + then throwAfter (n - 1) + else throw "Uh oh!"; +in + throwAfter 2 diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp index 0069285fb7e..7cb08af8aad 100644 --- a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp +++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -1,35 +1,27 @@ error: … while calling the 'foldl'' builtin - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: - 1| # Tests that the result of applying op is forced even if the value is never used 2| builtins.foldl' | ^ 3| (_: f: f null) … while calling anonymous lambda - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: - 2| builtins.foldl' 3| (_: f: f null) | ^ 4| null … from call site - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: - 2| builtins.foldl' 3| (_: f: f null) | ^ 4| null … while calling anonymous lambda - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: - 4| null 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] | ^ diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp index 5b60d253de3..9bbb251e14b 100644 --- a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp @@ -1,10 +1,8 @@ error: … while calling the 'fromTOML' builtin - at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: - 1| builtins.fromTOML '' | ^ 2| key = "value" - error: while parsing a TOML string: Dates and times are not supported + error: while parsing TOML: Dates and times are not supported diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp index 6d38608c0c5..1e465392744 100644 --- a/tests/functional/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'toString' builtin - at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: - 3| in 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) | ^ diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp new file mode 100644 index 00000000000..5d843d827c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp @@ -0,0 +1,38 @@ +error: + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:1: + 1| (x: x x) (x: x x) + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:2: + 1| (x: x x) (x: x x) + | ^ + 2| + + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:5: + 1| (x: x x) (x: x x) + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:11: + 1| (x: x x) (x: x x) + | ^ + 2| + + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: + 1| (x: x x) (x: x x) + | ^ + 2| + + (19997 duplicate frames omitted) + + error: stack overflow; max-call-depth exceeded + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: + 1| (x: x x) (x: x x) + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix new file mode 100644 index 00000000000..dd0a8bf2e76 --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix @@ -0,0 +1 @@ +(x: x x) (x: x x) diff --git a/tests/functional/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp index 24d68211837..d492f8bd2e4 100644 --- a/tests/functional/lang/eval-fail-list.err.exp +++ b/tests/functional/lang/eval-fail-list.err.exp @@ -1,10 +1,8 @@ error: … while evaluating one of the elements to concatenate - at /pwd/lang/eval-fail-list.nix:1:2: - 1| 8++1 | ^ 2| - error: value is an integer while a list was expected + error: expected a list but found an integer: 8 diff --git a/tests/functional/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp index 61fabf0d5f1..3b162fe1b60 100644 --- a/tests/functional/lang/eval-fail-missing-arg.err.exp +++ b/tests/functional/lang/eval-fail-missing-arg.err.exp @@ -1,16 +1,12 @@ error: … from call site - at /pwd/lang/eval-fail-missing-arg.nix:1:1: - 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} | ^ 2| error: function 'anonymous lambda' called without required argument 'y' - at /pwd/lang/eval-fail-missing-arg.nix:1:2: - 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} | ^ 2| diff --git a/tests/functional/lang/eval-fail-mutual-recursion.err.exp b/tests/functional/lang/eval-fail-mutual-recursion.err.exp new file mode 100644 index 00000000000..dc2e117661a --- /dev/null +++ b/tests/functional/lang/eval-fail-mutual-recursion.err.exp @@ -0,0 +1,57 @@ +error: + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:36:3: + 35| in + 36| throwAfterA true 10 + | ^ + 37| + + … while calling 'throwAfterA' + at /pwd/lang/eval-fail-mutual-recursion.nix:29:26: + 28| + 29| throwAfterA = recurse: n: + | ^ + 30| if n > 0 + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:31:10: + 30| if n > 0 + 31| then throwAfterA recurse (n - 1) + | ^ + 32| else if recurse + + (19 duplicate frames omitted) + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:33:10: + 32| else if recurse + 33| then throwAfterB true 10 + | ^ + 34| else throw "Uh oh!"; + + … while calling 'throwAfterB' + at /pwd/lang/eval-fail-mutual-recursion.nix:22:26: + 21| let + 22| throwAfterB = recurse: n: + | ^ + 23| if n > 0 + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:24:10: + 23| if n > 0 + 24| then throwAfterB recurse (n - 1) + | ^ + 25| else if recurse + + (19 duplicate frames omitted) + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:26:10: + 25| else if recurse + 26| then throwAfterA false 10 + | ^ + 27| else throw "Uh oh!"; + + (21 duplicate frames omitted) + + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-mutual-recursion.nix b/tests/functional/lang/eval-fail-mutual-recursion.nix new file mode 100644 index 00000000000..d090d3158a3 --- /dev/null +++ b/tests/functional/lang/eval-fail-mutual-recursion.nix @@ -0,0 +1,36 @@ +# Check that stack frame deduplication only affects consecutive intervals, and +# that they are reported independently of any preceding sections, even if +# they're indistinguishable. +# +# In terms of the current implementation, we check that we clear the set of +# "seen frames" after eliding a group of frames. +# +# Suppose we have: +# - 10 frames in a function A +# - 10 frames in a function B +# - 10 frames in a function A +# +# We want to output: +# - a few frames of A (skip the rest) +# - a few frames of B (skip the rest) +# - a few frames of A (skip the rest) +# +# If we implemented this in the naive manner, we'd instead get: +# - a few frames of A (skip the rest) +# - a few frames of B (skip the rest, _and_ skip the remaining frames of A) +let + throwAfterB = recurse: n: + if n > 0 + then throwAfterB recurse (n - 1) + else if recurse + then throwAfterA false 10 + else throw "Uh oh!"; + + throwAfterA = recurse: n: + if n > 0 + then throwAfterA recurse (n - 1) + else if recurse + then throwAfterB true 10 + else throw "Uh oh!"; +in + throwAfterA true 10 diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp index b290afb0aaa..fc81f7277e1 100644 --- a/tests/functional/lang/eval-fail-not-throws.err.exp +++ b/tests/functional/lang/eval-fail-not-throws.err.exp @@ -1,16 +1,12 @@ error: … in the argument of the not operator - at /pwd/lang/eval-fail-not-throws.nix:1:4: - 1| ! (throw "uh oh!") | ^ 2| … while calling the 'throw' builtin - at /pwd/lang/eval-fail-not-throws.nix:1:4: - 1| ! (throw "uh oh!") | ^ 2| diff --git a/tests/functional/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp index f0011c97fd5..e3531d352f3 100644 --- a/tests/functional/lang/eval-fail-path-slash.err.exp +++ b/tests/functional/lang/eval-fail-path-slash.err.exp @@ -1,7 +1,5 @@ error: path has a trailing slash - at /pwd/lang/eval-fail-path-slash.nix:6:12: - 5| # and https://nixos.org/nix-dev/2016-June/020829.html 6| /nix/store/ | ^ diff --git a/tests/functional/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp index af64133cb88..19380dc6536 100644 --- a/tests/functional/lang/eval-fail-recursion.err.exp +++ b/tests/functional/lang/eval-fail-recursion.err.exp @@ -1,16 +1,12 @@ error: … in the right operand of the update (//) operator - at /pwd/lang/eval-fail-recursion.nix:1:12: - 1| let a = {} // a; in a.foo | ^ 2| error: infinite recursion encountered - at /pwd/lang/eval-fail-recursion.nix:1:15: - 1| let a = {} // a; in a.foo | ^ 2| diff --git a/tests/functional/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp index e82cdac9865..292b3c3f33a 100644 --- a/tests/functional/lang/eval-fail-remove.err.exp +++ b/tests/functional/lang/eval-fail-remove.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-remove.nix:4:3: - 3| 4| body = (removeAttrs attrs ["x"]).x; | ^ 5| } error: attribute 'x' missing - at /pwd/lang/eval-fail-remove.nix:4:10: - 3| 4| body = (removeAttrs attrs ["x"]).x; | ^ diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp index 22b6166f803..b0b05cad737 100644 --- a/tests/functional/lang/eval-fail-scope-5.err.exp +++ b/tests/functional/lang/eval-fail-scope-5.err.exp @@ -1,35 +1,27 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-scope-5.nix:8:3: - 7| 8| body = f {}; | ^ 9| … from call site - at /pwd/lang/eval-fail-scope-5.nix:8:10: - 7| 8| body = f {}; | ^ 9| … while calling 'f' - at /pwd/lang/eval-fail-scope-5.nix:6:7: - 5| 6| f = {x ? y, y ? x}: x + y; | ^ 7| error: infinite recursion encountered - at /pwd/lang/eval-fail-scope-5.nix:6:12: - 5| 6| f = {x ? y, y ? x}: x + y; | ^ diff --git a/tests/functional/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp index 33a7e9491aa..3e3d71b15d0 100644 --- a/tests/functional/lang/eval-fail-seq.err.exp +++ b/tests/functional/lang/eval-fail-seq.err.exp @@ -1,16 +1,12 @@ error: … while calling the 'seq' builtin - at /pwd/lang/eval-fail-seq.nix:1:1: - 1| builtins.seq (abort "foo") 2 | ^ 2| … while calling the 'abort' builtin - at /pwd/lang/eval-fail-seq.nix:1:15: - 1| builtins.seq (abort "foo") 2 | ^ 2| diff --git a/tests/functional/lang/eval-fail-set-override.err.exp b/tests/functional/lang/eval-fail-set-override.err.exp index 71481683db1..9006ca4e677 100644 --- a/tests/functional/lang/eval-fail-set-override.err.exp +++ b/tests/functional/lang/eval-fail-set-override.err.exp @@ -1,4 +1,4 @@ error: … while evaluating the `__overrides` attribute - error: value is an integer while a set was expected + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp index 0d014050854..6dd646e11ec 100644 --- a/tests/functional/lang/eval-fail-set.err.exp +++ b/tests/functional/lang/eval-fail-set.err.exp @@ -1,7 +1,5 @@ error: undefined variable 'x' - at /pwd/lang/eval-fail-set.nix:1:3: - 1| 8.x | ^ 2| diff --git a/tests/functional/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp index 5c58be29a66..0457a826ea3 100644 --- a/tests/functional/lang/eval-fail-substring.err.exp +++ b/tests/functional/lang/eval-fail-substring.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'substring' builtin - at /pwd/lang/eval-fail-substring.nix:1:1: - 1| builtins.substring (builtins.sub 0 1) 1 "x" | ^ 2| diff --git a/tests/functional/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp index 4ffa2cf6dbc..d6b17be992f 100644 --- a/tests/functional/lang/eval-fail-to-path.err.exp +++ b/tests/functional/lang/eval-fail-to-path.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'toPath' builtin - at /pwd/lang/eval-fail-to-path.nix:1:1: - 1| builtins.toPath "foo/bar" | ^ 2| diff --git a/tests/functional/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp index 4e618c20394..ad267711be1 100644 --- a/tests/functional/lang/eval-fail-toJSON.err.exp +++ b/tests/functional/lang/eval-fail-toJSON.err.exp @@ -1,54 +1,47 @@ error: … while calling the 'toJSON' builtin - at /pwd/lang/eval-fail-toJSON.nix:1:1: - 1| builtins.toJSON { | ^ 2| a.b = [ … while evaluating attribute 'a' - at /pwd/lang/eval-fail-toJSON.nix:2:3: - 1| builtins.toJSON { 2| a.b = [ | ^ 3| true … while evaluating attribute 'b' - at /pwd/lang/eval-fail-toJSON.nix:2:3: - 1| builtins.toJSON { 2| a.b = [ | ^ 3| true … while evaluating list element at index 3 + at /pwd/lang/eval-fail-toJSON.nix:2:3: + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true … while evaluating attribute 'c' - at /pwd/lang/eval-fail-toJSON.nix:7:7: - 6| { 7| c.d = throw "hah no"; | ^ 8| } … while evaluating attribute 'd' - at /pwd/lang/eval-fail-toJSON.nix:7:7: - 6| { 7| c.d = throw "hah no"; | ^ 8| } … while calling the 'throw' builtin - at /pwd/lang/eval-fail-toJSON.nix:7:13: - 6| { 7| c.d = throw "hah no"; | ^ diff --git a/tests/functional/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp index 30db743c7ee..6e13a138eb7 100644 --- a/tests/functional/lang/eval-fail-undeclared-arg.err.exp +++ b/tests/functional/lang/eval-fail-undeclared-arg.err.exp @@ -1,16 +1,12 @@ error: … from call site - at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: - 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} | ^ 2| error: function 'anonymous lambda' called with unexpected argument 'y' - at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: - 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} | ^ 2| diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp index 811d01b0378..4326c965008 100644 --- a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -1,11 +1,14 @@ error: … while evaluating an attribute name - at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: - 4| in 5| attr.${key} | ^ 6| - error: value is a set while a string was expected + error: expected a string but found a set: { } + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + 4| in + 5| attr.${key} + | ^ + 6| diff --git a/tests/functional/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp index 3fc99be3efa..80aa17c6e1d 100644 --- a/tests/functional/lang/eval-okay-print.err.exp +++ b/tests/functional/lang/eval-okay-print.err.exp @@ -1 +1 @@ -trace: [ ] +trace: [ «thunk» ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-attrs.exp b/tests/functional/lang/eval-okay-repeated-empty-attrs.exp new file mode 100644 index 00000000000..d21e6db6b75 --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-attrs.exp @@ -0,0 +1 @@ +[ { } { } ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-attrs.nix b/tests/functional/lang/eval-okay-repeated-empty-attrs.nix new file mode 100644 index 00000000000..030a3b85c76 --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-attrs.nix @@ -0,0 +1,2 @@ +# Tests that empty attribute sets are not printed as `«repeated»`. +[ {} {} ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-list.exp b/tests/functional/lang/eval-okay-repeated-empty-list.exp new file mode 100644 index 00000000000..701fc7e2042 --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-list.exp @@ -0,0 +1 @@ +[ [ ] [ ] ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-list.nix b/tests/functional/lang/eval-okay-repeated-empty-list.nix new file mode 100644 index 00000000000..376c51be886 --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-list.nix @@ -0,0 +1 @@ +[ [] [] ] diff --git a/tests/functional/lang/eval-okay-substring-context.exp b/tests/functional/lang/eval-okay-substring-context.exp new file mode 100644 index 00000000000..2fe7f71fa11 --- /dev/null +++ b/tests/functional/lang/eval-okay-substring-context.exp @@ -0,0 +1 @@ +"okay" diff --git a/tests/functional/lang/eval-okay-substring-context.nix b/tests/functional/lang/eval-okay-substring-context.nix new file mode 100644 index 00000000000..d0ef70d4e67 --- /dev/null +++ b/tests/functional/lang/eval-okay-substring-context.nix @@ -0,0 +1,11 @@ +with builtins; + +let + + s = "${builtins.derivation { name = "test"; builder = "/bin/sh"; system = "x86_64-linux"; }}"; + +in + +if getContext s == getContext "${substring 0 0 s + unsafeDiscardStringContext s}" +then "okay" +else throw "empty substring should preserve context" diff --git a/tests/functional/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp index 6aace04b0f5..f48b4623a5e 100644 --- a/tests/functional/lang/eval-okay-substring.exp +++ b/tests/functional/lang/eval-okay-substring.exp @@ -1 +1 @@ -"ooxfoobarybarzobaabbc" +"ooxfoobarybarzobaabbc_bad" diff --git a/tests/functional/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix index 424af00d9b3..54c97e16263 100644 --- a/tests/functional/lang/eval-okay-substring.nix +++ b/tests/functional/lang/eval-okay-substring.nix @@ -19,3 +19,5 @@ substring 1 2 s + substring 3 1 s + "c" + substring 5 10 "perl" ++ "_" ++ substring 3 (-1) "tebbad" diff --git a/tests/functional/lang/framework.sh b/tests/functional/lang/framework.sh index 516bff8ad1d..9b886e9835a 100644 --- a/tests/functional/lang/framework.sh +++ b/tests/functional/lang/framework.sh @@ -16,7 +16,7 @@ function diffAndAcceptInner() { fi # Diff so we get a nice message - if ! diff --unified "$got" "$expectedOrEmpty"; then + if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then echo "FAIL: evaluation result of $testName not as expected" badDiff=1 fi diff --git a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp index 4fe6b7a1f0e..6c3a3510ccd 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:1:3 - at «stdin»:3:3: - 2| y = 456; 3| x = 789; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp index 3aba2891f62..fecdece208c 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:9:5 - at «stdin»:10:17: - 9| x = 789; 10| inherit (as) x; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp index 3aba2891f62..fecdece208c 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:9:5 - at «stdin»:10:17: - 9| x = 789; 10| inherit (as) x; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp index ff68446a1a8..f85ffea5135 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp @@ -1,7 +1,5 @@ error: attribute 'services.ssh.port' already defined at «stdin»:2:3 - at «stdin»:3:3: - 2| services.ssh.port = 22; 3| services.ssh.port = 23; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp index 512a499ca9b..98cea9dae20 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:6:12 - at «stdin»:7:12: - 6| inherit x; 7| inherit x; | ^ diff --git a/tests/functional/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp index 1d566fb3342..d7c7e023760 100644 --- a/tests/functional/lang/parse-fail-dup-formals.err.exp +++ b/tests/functional/lang/parse-fail-dup-formals.err.exp @@ -1,6 +1,4 @@ error: duplicate formal function argument 'x' - at «stdin»:1:8: - 1| {x, y, x}: x | ^ diff --git a/tests/functional/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp index f9fa7231292..b28d35950e6 100644 --- a/tests/functional/lang/parse-fail-eof-in-string.err.exp +++ b/tests/functional/lang/parse-fail-eof-in-string.err.exp @@ -1,7 +1,5 @@ error: syntax error, unexpected end of file, expecting '"' - at «stdin»:3:5: - 2| # Note that this file must not end with a newline. 3| a 1"$ | ^ diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp index 32f77679584..a4472156b29 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -1,7 +1,5 @@ error: attribute 'z' already defined at «stdin»:3:16 - at «stdin»:2:3: - 1| { 2| x.z = 3; | ^ diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp index 0437cd50c8a..ead1f0dbd82 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -1,7 +1,5 @@ error: attribute 'y' already defined at «stdin»:3:9 - at «stdin»:2:3: - 1| { 2| x.y.y = 3; | ^ diff --git a/tests/functional/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp index 634a04aaa4b..6ba39d88466 100644 --- a/tests/functional/lang/parse-fail-patterns-1.err.exp +++ b/tests/functional/lang/parse-fail-patterns-1.err.exp @@ -1,7 +1,5 @@ error: duplicate formal function argument 'args' - at «stdin»:1:1: - 1| args@{args, x, y, z}: x | ^ 2| diff --git a/tests/functional/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp index 167d01e85d3..d8875a6a57a 100644 --- a/tests/functional/lang/parse-fail-regression-20060610.err.exp +++ b/tests/functional/lang/parse-fail-regression-20060610.err.exp @@ -1,7 +1,5 @@ error: undefined variable 'gcc' - at «stdin»:8:12: - 7| 8| body = ({ | ^ diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 77c96bbd23a..a58d8dca4c5 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,7 +1,5 @@ error: syntax error, unexpected ':', expecting '}' - at «stdin»:3:13: - 2| 3| f = {x, y : | ^ diff --git a/tests/functional/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp index 48e88747faf..3d143d9aff4 100644 --- a/tests/functional/lang/parse-fail-undef-var.err.exp +++ b/tests/functional/lang/parse-fail-undef-var.err.exp @@ -1,7 +1,5 @@ error: undefined variable 'y' - at «stdin»:1:4: - 1| x: y | ^ 2| diff --git a/tests/functional/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp index 6087479a37e..e83abdb9ede 100644 --- a/tests/functional/lang/parse-fail-utf8.err.exp +++ b/tests/functional/lang/parse-fail-utf8.err.exp @@ -1,6 +1,4 @@ error: syntax error, unexpected invalid token, expecting end of file - at «stdin»:1:5: - 1| 123 | ^ diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 35cd48002e1..a6d44529111 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -2,6 +2,7 @@ nix_tests = \ test-infra.sh \ init.sh \ flakes/flakes.sh \ + flakes/develop.sh \ flakes/run.sh \ flakes/mercurial.sh \ flakes/circular.sh \ @@ -49,7 +50,8 @@ nix_tests = \ optimise-store.sh \ substitute-with-invalid-ca.sh \ signing.sh \ - hash.sh \ + hash-convert.sh \ + hash-path.sh \ gc-non-blocking.sh \ check.sh \ nix-shell.sh \ @@ -74,6 +76,7 @@ nix_tests = \ build-remote-trustless-should-fail-0.sh \ build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ + impure-eval.sh \ pure-eval.sh \ eval.sh \ repl.sh \ @@ -128,18 +131,19 @@ nix_tests = \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh \ - impure-env.sh + impure-env.sh \ + debugger.sh ifeq ($(HAVE_LIBCPUID), 1) - nix_tests += compute-levels.sh + nix_tests += compute-levels.sh endif ifeq ($(ENABLE_BUILD), yes) - nix_tests += test-libstoreconsumer.sh + nix_tests += test-libstoreconsumer.sh - ifeq ($(BUILD_SHARED_LIBS), 1) - nix_tests += plugins.sh - endif + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif endif $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index 1481b9b36da..1ccc21d0bce 100644 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -15,7 +15,7 @@ nix-build dependencies.nix --no-out-link --compress-build-log [ "$(nix-store -l $path)" = FOO ] # test whether empty logs work fine with `nix log`. -builder="$(mktemp)" +builder="$(realpath "$(mktemp)")" echo -e "#!/bin/sh\nmkdir \$out" > "$builder" outp="$(nix-build -E \ 'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \ diff --git a/tests/functional/nix-channel.sh b/tests/functional/nix-channel.sh index b5d93500409..e4131cee13c 100644 --- a/tests/functional/nix-channel.sh +++ b/tests/functional/nix-channel.sh @@ -29,7 +29,8 @@ unset NIX_CONFIG # Create a channel. rm -rf $TEST_ROOT/foo mkdir -p $TEST_ROOT/foo -nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix)) +drvPath=$(nix-instantiate dependencies.nix) +nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r "$drvPath") rm -rf $TEST_ROOT/nixexprs mkdir -p $TEST_ROOT/nixexprs cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ @@ -64,3 +65,5 @@ grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] +# Test evaluation through a channel symlink (#9882). +#nix-instantiate '' diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh new file mode 100644 index 00000000000..cc8314ff7fa --- /dev/null +++ b/tests/functional/nix-copy-ssh-common.sh @@ -0,0 +1,70 @@ +proto=$1 +shift +(( $# == 0 )) + +clearStore +clearCache + +mkdir -p $TEST_ROOT/stores + +# Create path to copy back and forth +outPath=$(nix-build --no-out-link dependencies.nix) + +storeQueryParam="store=${NIX_STORE_DIR}" + +realQueryParam () { + echo "real=$1$NIX_STORE_DIR" +} + +remoteRoot="$TEST_ROOT/stores/$proto" + +clearRemoteStore () { + chmod -R u+w "$remoteRoot" || true + rm -rf "$remoteRoot" +} + +clearRemoteStore + +remoteStore="${proto}://localhost?${storeQueryParam}&remote-store=${remoteRoot}%3f${storeQueryParam}%26$(realQueryParam "$remoteRoot")" + +# Copy to store + +args=() +if [[ "$proto" == "ssh-ng" ]]; then + # TODO investigate discrepancy + args+=(--no-check-sigs) +fi + +[ ! -f ${remoteRoot}${outPath}/foobar ] +nix copy "${args[@]}" --to "$remoteStore" $outPath +[ -f ${remoteRoot}${outPath}/foobar ] + +# Copy back from store + +clearStore + +[ ! -f $outPath/foobar ] +nix copy --no-check-sigs --from "$remoteStore" $outPath +[ -f $outPath/foobar ] + +# Check --substitute-on-destination, avoid corrupted store + +clearRemoteStore + +corruptedRoot=$TEST_ROOT/stores/corrupted +corruptedStore="${corruptedRoot}?${storeQueryParam}&$(realQueryParam "$corruptedRoot")" + +# Copy it to the corrupted store +nix copy --no-check-sigs "$outPath" --to "$corruptedStore" + +# Corrupt it in there +corruptPath="${corruptedRoot}${outPath}" +chmod +w "$corruptPath" +echo "not supposed to be here" > "$corruptPath/foobarbaz" +chmod -w "$corruptPath" + +# Copy from the corrupted store with the regular store as a +# substituter. It must use the substituter not the source store in +# order to avoid errors. +NIX_CONFIG=$(echo -e "substituters = local\nrequire-sigs = false") \ + nix copy --no-check-sigs --from "$corruptedStore" --to "$remoteStore" --substitute-on-destination "$outPath" diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 463b5e0c4bb..62e99cd2477 100644 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -1,18 +1,14 @@ source common.sh -clearStore -clearCache +source nix-copy-ssh-common.sh "ssh-ng" -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" +clearStore +clearRemoteStore outPath=$(nix-build --no-out-link dependencies.nix) -nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "$remoteStore" # Regression test for https://github.com/NixOS/nix/issues/6253 -nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & -nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs - -[ -f $remoteRoot$outPath/foobar ] +nix copy --to "$remoteStore" $outPath --no-check-sigs & +nix copy --to "$remoteStore" $outPath --no-check-sigs diff --git a/tests/functional/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh index eb801548d2f..12e8346bcfd 100644 --- a/tests/functional/nix-copy-ssh.sh +++ b/tests/functional/nix-copy-ssh.sh @@ -1,20 +1,3 @@ source common.sh -clearStore -clearCache - -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" - -outPath=$(nix-build --no-out-link dependencies.nix) - -nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $remoteRoot$outPath/foobar ] - -clearStore - -nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $outPath/foobar ] +source nix-copy-ssh-common.sh "ssh" diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 0969a461568..271ed89b73e 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -47,11 +47,11 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' +nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L nix profile list --json | jq . # FIXME: path flakes are not currently locked -#nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' +#nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) @@ -60,9 +60,8 @@ nix profile history | grep "packages.$system.default: ∅ -> 1.0" nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' # Test XDG Base Directories support - export NIX_CONFIG="use-xdg-base-directories = true" -nix profile remove 1 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' nix profile install $flake1Dir [[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]] unset NIX_CONFIG @@ -70,7 +69,7 @@ unset NIX_CONFIG # Test upgrading a package. printf NixOS > $flake1Dir/who printf 2.0 > $flake1Dir/version -nix profile upgrade 1 +nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" @@ -83,7 +82,7 @@ nix profile rollback # Test uninstall. [ -e $TEST_HOME/.nix-profile/bin/foo ] -nix profile remove 0 +nix profile remove foo 2>&1 | grep 'removed 1 packages' (! [ -e $TEST_HOME/.nix-profile/bin/foo ]) nix profile history | grep 'foo: 1.0 -> ∅' nix profile diff-closures | grep 'Version 3 -> 4' @@ -91,10 +90,18 @@ nix profile diff-closures | grep 'Version 3 -> 4' # Test installing a non-flake package. nix profile install --file ./simple.nix '' [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] -nix profile remove 1 +nix profile remove simple 2>&1 | grep 'removed 1 packages' nix profile install $(nix-build --no-out-link ./simple.nix) [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +# Test packages with same name from different sources +mkdir $TEST_ROOT/simple-too +cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too +nix profile install --file $TEST_ROOT/simple-too/simple.nix '' +nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple-1' +nix profile remove simple 2>&1 | grep 'removed 1 packages' +nix profile remove simple-1 2>&1 | grep 'removed 1 packages' + # Test wipe-history. nix profile wipe-history [[ $(nix profile history | grep Version | wc -l) -eq 1 ]] @@ -102,11 +109,11 @@ nix profile wipe-history # Test upgrade to CA package. printf true > $flake1Dir/ca.nix printf 3.0 > $flake1Dir/version -nix profile upgrade 0 +nix profile upgrade flake1 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" # Test new install of CA package. -nix profile remove 0 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' printf 4.0 > $flake1Dir/version printf Utrecht > $flake1Dir/who nix profile install $flake1Dir @@ -114,26 +121,27 @@ nix profile install $flake1Dir [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] # Override the outputs. -nix profile remove 0 1 +nix profile remove simple flake1 nix profile install "$flake1Dir^*" [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] [ -e $TEST_HOME/.nix-profile/share/man ] [ -e $TEST_HOME/.nix-profile/include ] printf Nix > $flake1Dir/who -nix profile upgrade 0 +nix profile list +nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] [ -e $TEST_HOME/.nix-profile/share/man ] [ -e $TEST_HOME/.nix-profile/include ] -nix profile remove 0 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' nix profile install "$flake1Dir^man" (! [ -e $TEST_HOME/.nix-profile/bin/hello ]) [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) # test priority -nix profile remove 0 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' # Make another flake. flake2Dir=$TEST_ROOT/flake2 @@ -187,3 +195,12 @@ nix profile install $flake2Dir --priority 0 clearProfiles nix profile install $(nix build $flake1Dir --no-link --print-out-paths) expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default" + +# Test upgrading from profile version 2. +clearProfiles +mkdir -p $TEST_ROOT/import-profile +outPath=$(nix build --no-link --print-out-paths $flake1Dir/flake.nix^out) +printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > $TEST_ROOT/import-profile/manifest.json +nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile) --no-link +nix profile list | grep -A4 'Name:.*hello' | grep "Store paths:.*$outPath" +nix profile remove hello 2>&1 | grep 'removed 1 packages, kept 0 packages' diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index 13403fadb82..04c83138e7c 100644 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -118,10 +118,10 @@ diff $TEST_ROOT/dev-env{,2}.json # Ensure `nix print-dev-env --json` contains variable assignments. [[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]] -# Run tests involving `source <(nix print-dev-inv)` in subshells to avoid modifying the current +# Run tests involving `source <(nix print-dev-env)` in subshells to avoid modifying the current # environment. -set +u # FIXME: Make print-dev-env `set -u` compliant (issue #7951) +set -u # Ensure `source <(nix print-dev-env)` modifies the environment. ( diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 5c7bfde46ba..dc80f8b5555 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -19,18 +19,7 @@ else fi # Test import-from-derivation through the daemon. -[[ $(nix eval --impure --raw --expr ' - with import ./config.nix; - import ( - mkDerivation { - name = "foo"; - bla = import ./dependencies.nix {}; - buildCommand = " - echo \\\"hi\\\" > $out - "; - } - ) -') = hi ]] +[[ $(nix eval --impure --raw --file ./ifd.nix) = hi ]] storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 1b779c1f56d..4938c226730 100644 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -146,29 +146,86 @@ echo "$replResult" | grepQuiet -s afterChange # Normal output should print attributes in lexicographical order non-recursively testReplResponseNoRegex ' { a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } -' '{ a = { ... }; l = [ ... ]; n = 1234; s = "string"; x = { ... }; }' +' \ +'{ + a = { ... }; + l = [ ... ]; + n = 1234; + s = "string"; + x = { ... }; +} +' # Same for lists, but order is preserved testReplResponseNoRegex ' [ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] -' '[ 42 1 "thingy" { ... } [ ... ] ]' +' \ +'[ + 42 + 1 + "thingy" + { ... } + [ ... ] +] +' # Same for let expressions testReplResponseNoRegex ' let x = { y = { a = 1; }; inherit x; }; in x -' '{ x = { ... }; y = { ... }; }' +' \ +'{ + x = { ... }; + y = { ... }; +} +' # The :p command should recursively print sets, but prevent infinite recursion testReplResponseNoRegex ' :p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } -' '{ a = { b = 2; }; n = 1234; s = "string"; x = { y = { z = { y = «repeated»; }; }; }; }' +' \ +'{ + a = { b = 2; }; + n = 1234; + s = "string"; + x = { + y = { + z = { + y = «repeated»; + }; + }; + }; +} +' # Same for lists testReplResponseNoRegex ' :p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] -' '[ 42 1 "thingy" { a = 1; b = { a = 1; b = «repeated»; }; } [ 1 2 3 ] ]' +' \ +'[ + 42 + 1 + "thingy" + { + a = 1; + b = { + a = 1; + b = «repeated»; + }; + } + [ + 1 + 2 + 3 + ] +] +' # Same for let expressions testReplResponseNoRegex ' :p let x = { y = { a = 1; }; inherit x; }; in x -' '{ x = { x = «repeated»; y = { a = 1; }; }; y = «repeated»; }' +' \ +'{ + x = «repeated»; + y = { a = 1 }; +} +' diff --git a/tests/functional/search.sh b/tests/functional/search.sh index 8742f873663..d9c7a75da48 100644 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -17,12 +17,15 @@ clearCache # Multiple arguments will not exist (( $(nix search -f search.nix '' hello broken | wc -l) == 0 )) +# No regex should return an error +(( $(nix search -f search.nix '' | wc -l) == 0 )) + ## Search expressions # Check that empty search string matches all -nix search -f search.nix '' |grepQuiet foo -nix search -f search.nix '' |grepQuiet bar -nix search -f search.nix '' |grepQuiet hello +nix search -f search.nix '' ^ | grepQuiet foo +nix search -f search.nix '' ^ | grepQuiet bar +nix search -f search.nix '' ^ | grepQuiet hello ## Tests for multiple regex/match highlighting @@ -39,8 +42,8 @@ e=$'\x1b' # grep doesn't support \e, \033 or even \x1b (( $(nix search -f search.nix '' 'b' | grep -Eo "$e\[32;1mb$e\[(0|0;1)m" | wc -l) == 3 )) ## Tests for --exclude -(( $(nix search -f search.nix -e hello | grep -c hello) == 0 )) +(( $(nix search -f search.nix ^ -e hello | grep -c hello) == 0 )) -(( $(nix search -f search.nix foo --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) -(( $(nix search -f search.nix foo -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) -[[ $(nix search -f search.nix -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] +(( $(nix search -f search.nix foo ^ --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) +(( $(nix search -f search.nix foo ^ -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) +[[ $(nix search -f search.nix '' ^ -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index 3fdd3501d62..dfe66ef93c8 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -23,4 +23,20 @@ with import ./config.nix; chmod +x $dev/bin/hello2 ''; }; + + salve-mundi = mkDerivation { + name = "salve-mundi"; + outputs = [ "out" ]; + meta.outputsToInstall = [ "out" ]; + buildCommand = + '' + mkdir -p $out/bin + + cat > $out/bin/hello < {repo.path}/test-case \ + && echo lutyabrook > {repo.path}/new-york-state \ + && {repo.git} add test-case new-york-state \ + && {repo.git} commit -m 'commit1' + """) + + # memoize the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath" + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/new-york-state + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev" + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/http-simple/default.nix b/tests/nixos/fetch-git/test-cases/http-simple/default.nix new file mode 100644 index 00000000000..dcab8067e59 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/http-simple/default.nix @@ -0,0 +1,39 @@ +{ config, ... }: +{ + description = "can fetch a git repo via http"; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' + """) + + # save the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath" + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev" + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix new file mode 100644 index 00000000000..57561e74b3e --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix @@ -0,0 +1,57 @@ +{ + description = "can fetch the same repo shallowly and non-shallowly"; + script = '' + # create branch1 off of main + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + \ + && {repo.git} push origin --all + """) + + # save the revision + mainRev = client.succeed(f""" + {repo.git} rev-parse main + """).strip() + + # fetch shallowly + revCountShallow = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = true; + }}).revCount + ' + """).strip() + # ensure the revCount is 0 + assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}" + + # fetch non-shallowly + revCountNonShallow = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = false; + }}).revCount + ' + """).strip() + # ensure the revCount is 1 + assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}" + + # fetch shallowly again + revCountShallow2 = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = true; + }}).revCount + ' + """).strip() + # ensure the revCount is 0 + assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix new file mode 100644 index 00000000000..456ee83416e --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix @@ -0,0 +1,40 @@ +{ + description = "ensure that ref gets ignored when shallow=true is set"; + script = '' + # create branch1 off of main + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + \ + && {repo.git} checkout -b branch1 main \ + && echo bangkok > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit2' \ + \ + && {repo.git} push origin --all + """) + + # save the revisions + mainRev = client.succeed(f""" + {repo.git} rev-parse main + """).strip() + branch1Rev = client.succeed(f""" + {repo.git} rev-parse branch1 + """).strip() + + # Ensure that ref gets ignored when fetching shallowly. + # This would fail if the ref was respected, as branch1Rev is not on main. + client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{branch1Rev}"; + ref = "main"; + shallow = true; + }}) + ' + """) + + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix new file mode 100644 index 00000000000..979512af982 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix @@ -0,0 +1,52 @@ +{ + description = "can fetch a git repo via ssh using shallow=1"; + script = '' + # add a file to the repo + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' + """) + + # memoize the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin-ssh main + """) + + fetchGit_expr = f""" + builtins.fetchGit {{ + url = "{repo.remote_ssh}"; + rev = "{rev1}"; + shallow = true; + }} + """ + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).outPath' + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).rev' + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + + # check if revCount is 1 + revCount1 = client.succeed(f""" + nix eval --impure --expr '({fetchGit_expr}).revCount' + """).strip() + print(f"revCount1: {revCount1}") + assert revCount1 == '0', f"rev count is not 0 but {revCount1}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix new file mode 100644 index 00000000000..f5fba169846 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix @@ -0,0 +1,43 @@ +{ config, ... }: +{ + description = "can fetch a git repo via ssh"; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' + """) + + # save the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin-ssh main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit "{repo.remote_ssh}").outPath + ' + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit "{repo.remote_ssh}").rev + ' + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix new file mode 100644 index 00000000000..e9f4adcc1d3 --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix @@ -0,0 +1,75 @@ +{ lib, ... }: +let + inherit (lib) + mkIf + mkOption + types + ; + + boolPyLiteral = b: if b then "True" else "False"; + + testCaseExtension = { config, ... }: { + options = { + repo.enable = mkOption { + type = types.bool; + default = true; + description = "Whether to provide a repo variable - automatic repo creation."; + }; + repo.private = mkOption { + type = types.bool; + default = false; + description = "Whether the repo should be private."; + }; + }; + config = mkIf config.repo.enable { + setupScript = '' + repo = Repo("${config.name}", private=${boolPyLiteral config.repo.private}) + ''; + }; + }; +in +{ + options = { + testCases = mkOption { + type = types.listOf (types.submodule testCaseExtension); + }; + }; + config = { + setupScript = '' + def boolToJSON(b): + return "true" if b else "false" + + class Repo: + """ + A class to create a git repository on the gitea server and locally. + """ + def __init__(self, name, private=False): + self.name = name + self.path = "/tmp/repos/" + name + self.remote = "http://gitea:3000/test/" + name + self.remote_ssh = "ssh://gitea/root/" + name + self.git = f"git -C {self.path}" + self.private = private + self.create() + + def create(self): + # create ssh remote repo + gitea.succeed(f""" + git init --bare -b main /root/{self.name} + """) + # create http remote repo + gitea.succeed(f""" + curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main", "private": {boolToJSON(self.private)}}}' )} + """) + # setup git remotes on client + client.succeed(f""" + mkdir -p {self.path} \ + && git init -b main {self.path} \ + && {self.git} remote add origin {self.remote} \ + && {self.git} remote add origin-ssh root@gitea:{self.name} + """) + ''; + }; +} diff --git a/tests/nixos/fetch-git/testsupport/gitea.nix b/tests/nixos/fetch-git/testsupport/gitea.nix new file mode 100644 index 00000000000..cf87bb4662d --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/gitea.nix @@ -0,0 +1,102 @@ +{ lib, nixpkgs, system, pkgs, ... }: let + clientPrivateKey = pkgs.writeText "id_ed25519" '' + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQAAAJAwVQ5VMFUO + VQAAAAtzc2gtZWQyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQ + AAAEB7lbfkkdkJoE+4TKHPdPQWBKLSx+J54Eg8DaTr+3KoSlt5a8eH8BYZYjoQhzXGVKKH + Je1pw1D0p7O2Vb9VTLzBAAAACGJmb0BtaW5pAQIDBAU= + -----END OPENSSH PRIVATE KEY----- + ''; + + clientPublicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFt5a8eH8BYZYjoQhzXGVKKHJe1pw1D0p7O2Vb9VTLzB"; + +in { + imports = [ + ../testsupport/setup.nix + ../testsupport/gitea-repo.nix + ]; + nodes = { + gitea = { pkgs, ... }: { + services.gitea.enable = true; + services.gitea.settings.service.DISABLE_REGISTRATION = true; + services.gitea.settings.log.LEVEL = "Info"; + services.gitea.settings.database.LOG_SQL = false; + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 3000 ]; + environment.systemPackages = [ pkgs.git pkgs.gitea ]; + + users.users.root.openssh.authorizedKeys.keys = [clientPublicKey]; + + # TODO: remove this after updating to nixos-23.11 + nixpkgs.pkgs = lib.mkForce (import nixpkgs { + inherit system; + config.permittedInsecurePackages = [ + "gitea-1.19.4" + ]; + }); + }; + client = { pkgs, ... }: { + environment.systemPackages = [ pkgs.git ]; + }; + }; + defaults = { pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + }; + + setupScript = '' + import shlex + + gitea.wait_for_unit("gitea.service") + + gitea_admin = "test" + gitea_admin_password = "test123test" + + gitea.succeed(f""" + gitea --version >&2 + su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create \ + --username {gitea_admin} --password {gitea_admin_password} --email test@client' + """) + + client.wait_for_unit("multi-user.target") + gitea.wait_for_open_port(3000) + + gitea_admin_token = gitea.succeed(f""" + curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/users/test/tokens \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d {shlex.quote( '{"name":"token", "scopes":["all"]}' )} \ + | jq -r '.sha1' + """).strip() + + client.succeed(f""" + echo "http://{gitea_admin}:{gitea_admin_password}@gitea:3000" >~/.git-credentials-admin + git config --global credential.helper 'store --file ~/.git-credentials-admin' + git config --global user.email "test@client" + git config --global user.name "Test User" + git config --global gc.autodetach 0 + git config --global gc.auto 0 + """) + + # add client's private key to ~/.ssh + client.succeed(""" + mkdir -p ~/.ssh + chmod 700 ~/.ssh + cat ${clientPrivateKey} >~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + """) + + client.succeed(""" + echo "Host gitea" >>~/.ssh/config + echo " StrictHostKeyChecking no" >>~/.ssh/config + echo " UserKnownHostsFile /dev/null" >>~/.ssh/config + echo " User root" >>~/.ssh/config + """) + + # ensure ssh from client to gitea works + client.succeed(""" + ssh root@gitea true + """) + + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix new file mode 100644 index 00000000000..a81d5614b44 --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/setup.nix @@ -0,0 +1,91 @@ +{ lib, config, extendModules, ... }: +let + inherit (lib) + mkOption + types + ; + + indent = lib.replaceStrings ["\n"] ["\n "]; + + execTestCase = testCase: '' + + ### TEST ${testCase.name}: ${testCase.description} ### + + with subtest("${testCase.description}"): + # Setup + ${indent testCase.setupScript} + + # Test + ${indent testCase.script} + ''; +in +{ + + options = { + setupScript = mkOption { + type = types.lines; + description = '' + Python code that runs before the main test. + + Variables defined by this code will be available in the test. + ''; + default = ""; + }; + testCases = mkOption { + description = '' + The test cases. See `testScript`. + ''; + type = types.listOf (types.submodule { + options.name = mkOption { + type = types.str; + description = '' + The name of the test case. + + A repository with that name will be set up on the gitea server and locally. + ''; + }; + options.description = mkOption { + type = types.str; + description = '' + A description of the test case. + ''; + }; + options.setupScript = mkOption { + type = types.lines; + description = '' + Python code that runs before the test case. + ''; + default = ""; + }; + options.script = mkOption { + type = types.lines; + description = '' + Python code that runs the test. + + Variables defined by the global `setupScript`, as well as `testCases.*.setupScript` will be available here. + ''; + }; + }); + }; + }; + + config = { + nodes.client = { + environment.variables = { + _NIX_FORCE_HTTP = "1"; + }; + nix.settings.experimental-features = ["nix-command" "flakes"]; + }; + setupScript = '' + ''; + testScript = '' + start_all(); + + ${config.setupScript} + + ### SETUP COMPLETE ### + + ${lib.concatStringsSep "\n" (map execTestCase config.testCases)} + ''; + }; +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index fa563d4bd54..488830a8e98 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -144,7 +144,7 @@ in virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = "experimental-features = nix-command flakes"; - networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = + networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} = [ "channels.nixos.org" "api.github.com" "github.com" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 2981cc2b80d..7db5197aa8c 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -36,7 +36,7 @@ in { server = { config, pkgs, ... }: { services.openssh.enable = true; - services.openssh.permitRootLogin = "yes"; + services.openssh.settings.PermitRootLogin = "yes"; users.users.root.password = "foobar"; virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgB pkgC ]; diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index cef62e95bb2..00505d11421 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -84,8 +84,8 @@ in client = { lib, nodes, pkgs, ... }: { networking.useDHCP = false; networking.nameservers = [ - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address + (lib.head nodes.http_dns.networking.interfaces.eth1.ipv6.addresses).address + (lib.head nodes.http_dns.networking.interfaces.eth1.ipv4.addresses).address ]; networking.interfaces.eth1.ipv6.addresses = [ { address = "fd21::10"; prefixLength = 64; } diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index b59dde9bf1d..926ec00fead 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -1,4 +1,4 @@ -{ config, lib, hostPkgs, ... }: +test@{ config, lib, hostPkgs, ... }: let pkgs = config.nodes.client.nixpkgs.pkgs; @@ -28,81 +28,97 @@ let in { - name = "remote-builds-ssh-ng"; - - nodes = - { builder = - { config, pkgs, ... }: - { services.openssh.enable = true; - virtualisation.writableStore = true; - nix.settings.sandbox = true; - nix.settings.substituters = lib.mkForce [ ]; - }; - - client = - { config, lib, pkgs, ... }: - { nix.settings.max-jobs = 0; # force remote building - nix.distributedBuilds = true; - nix.buildMachines = - [ { hostName = "builder"; + options = { + builders.config = lib.mkOption { + type = lib.types.deferredModule; + description = '' + Configuration to add to the builder nodes. + ''; + default = { }; + }; + }; + + config = { + name = lib.mkDefault "remote-builds-ssh-ng"; + + nodes = + { + builder = + { config, pkgs, ... }: + { + imports = [ test.config.builders.config ]; + services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { + nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [{ + hostName = "builder"; sshUser = "root"; sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; maxJobs = 1; protocol = "ssh-ng"; - } - ]; - virtualisation.writableStore = true; - virtualisation.additionalPaths = [ config.system.build.extraUtils ]; - nix.settings.substituters = lib.mkForce [ ]; - programs.ssh.extraConfig = "ConnectTimeout 30"; - }; - }; + }]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; - testScript = { nodes }: '' - # fmt: off - import subprocess + testScript = { nodes }: '' + # fmt: off + import subprocess - start_all() + start_all() - # Create an SSH key on the client. - subprocess.run([ - "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" - ], capture_output=True, check=True) - client.succeed("mkdir -p -m 700 /root/.ssh") - client.copy_from_host("key", "/root/.ssh/id_ed25519") - client.succeed("chmod 600 /root/.ssh/id_ed25519") + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Install the SSH key on the builder. - client.wait_for_unit("network.target") - builder.succeed("mkdir -p -m 700 /root/.ssh") - builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") - builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - # Perform a build - out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + # Perform a build + out = client.succeed("nix-build ${expr nodes.client 1} 2> build-output") - # Verify that the build was done on the builder - builder.succeed(f"test -e {out.strip()}") + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") - # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix - buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") - print(buildOutput) + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) - # Make sure that we get the expected build output - client.succeed("grep -qF Hello build-output") + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") - # We don't want phase reporting in the build output - client.fail("grep -qF '@nix' build-output") + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") - # Get the log file - client.succeed(f"nix-store --read-log {out.strip()} > log-output") - # Prefix the log lines to avoid nix intercepting lines starting with @nix - logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") - print(logOutput) + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) - # Check that we get phase reporting in the log file - client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") - ''; + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; + }; } diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1c96cc78724..1661203ec16 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -1,6 +1,6 @@ # Test Nix's remote build feature. -{ config, lib, hostPkgs, ... }: +test@{ config, lib, hostPkgs, ... }: let pkgs = config.nodes.client.nixpkgs.pkgs; @@ -8,7 +8,9 @@ let # The configuration of the remote builders. builder = { config, pkgs, ... }: - { services.openssh.enable = true; + { + imports = [ test.config.builders.config ]; + services.openssh.enable = true; virtualisation.writableStore = true; nix.settings.sandbox = true; @@ -35,77 +37,94 @@ let in { - name = "remote-builds"; - - nodes = - { builder1 = builder; - builder2 = builder; - - client = - { config, lib, pkgs, ... }: - { nix.settings.max-jobs = 0; # force remote building - nix.distributedBuilds = true; - nix.buildMachines = - [ { hostName = "builder1"; - sshUser = "root"; - sshKey = "/root/.ssh/id_ed25519"; - system = "i686-linux"; - maxJobs = 1; - } - { hostName = "builder2"; - sshUser = "root"; - sshKey = "/root/.ssh/id_ed25519"; - system = "i686-linux"; - maxJobs = 1; - } - ]; - virtualisation.writableStore = true; - virtualisation.additionalPaths = [ config.system.build.extraUtils ]; - nix.settings.substituters = lib.mkForce [ ]; - programs.ssh.extraConfig = "ConnectTimeout 30"; - }; + options = { + builders.config = lib.mkOption { + type = lib.types.deferredModule; + description = '' + Configuration to add to the builder nodes. + ''; + default = { }; }; - - testScript = { nodes }: '' - # fmt: off - import subprocess - - start_all() - - # Create an SSH key on the client. - subprocess.run([ - "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" - ], capture_output=True, check=True) - client.succeed("mkdir -p -m 700 /root/.ssh") - client.copy_from_host("key", "/root/.ssh/id_ed25519") - client.succeed("chmod 600 /root/.ssh/id_ed25519") - - # Install the SSH key on the builders. - client.wait_for_unit("network.target") - for builder in [builder1, builder2]: - builder.succeed("mkdir -p -m 700 /root/.ssh") - builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") - builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - - # Perform a build and check that it was performed on the builder. - out = client.succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", - "grep -q Hello build-output" - ) - builder1.succeed(f"test -e {out}") - - # And a parallel build. - paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') - out1, out2 = paths.split() - builder1.succeed(f"test -e {out1} -o -e {out2}") - builder2.succeed(f"test -e {out1} -o -e {out2}") - - # And a failing build. - client.fail("nix-build ${expr nodes.client.config 5}") - - # Test whether the build hook automatically skips unavailable builders. - builder1.block() - client.succeed("nix-build ${expr nodes.client.config 4}") - ''; + }; + + config = { + name = lib.mkDefault "remote-builds"; + + nodes = + { + builder1 = builder; + builder2 = builder; + + client = + { config, lib, pkgs, ... }: + { + nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ + { + hostName = "builder1"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + { + hostName = "builder2"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + for builder in [builder1, builder2]: + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build and check that it was performed on the builder. + out = client.succeed( + "nix-build ${expr nodes.client 1} 2> build-output", + "grep -q Hello build-output" + ) + builder1.succeed(f"test -e {out}") + + # And a parallel build. + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client 2})\!out $(nix-instantiate ${expr nodes.client 3})\!out') + out1, out2 = paths.split() + builder1.succeed(f"test -e {out1} -o -e {out2}") + builder2.succeed(f"test -e {out1} -o -e {out2}") + + # And a failing build. + client.fail("nix-build ${expr nodes.client 5}") + + # Test whether the build hook automatically skips unavailable builders. + builder1.block() + client.succeed("nix-build ${expr nodes.client 4}") + ''; + }; } diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index 6e8d884a01b..04f3590e1d8 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -108,7 +108,7 @@ in flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json ''; environment.systemPackages = [ pkgs.jq ]; - networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} = + networking.hosts.${(builtins.head nodes.sourcehut.networking.interfaces.eth1.ipv4.addresses).address} = [ "git.sr.ht" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk index 12a76206ace..0501de33c43 100644 --- a/tests/unit/libexpr-support/local.mk +++ b/tests/unit/libexpr-support/local.mk @@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \ libstore-test-support libutil-test-support \ libexpr libstore libutil -libexpr-test-support_LDFLAGS := -pthread -lrapidcheck +libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc index 81498f65a3f..7b32b320bac 100644 --- a/tests/unit/libexpr/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -12,33 +12,33 @@ namespace nix { TEST_F(ErrorTraceTest, TraceBuilder) { ASSERT_THROW( - state.error("Not much").debugThrow(), + state.error("puppy").debugThrow(), EvalError ); ASSERT_THROW( - state.error("Not much").withTrace(noPos, "No more").debugThrow(), + state.error("puppy").withTrace(noPos, "doggy").debugThrow(), EvalError ); ASSERT_THROW( try { try { - state.error("Not much").withTrace(noPos, "No more").debugThrow(); + state.error("puppy").withTrace(noPos, "doggy").debugThrow(); } catch (Error & e) { - e.addTrace(state.positions[noPos], "Something", ""); + e.addTrace(state.positions[noPos], "beans"); throw; } } catch (BaseError & e) { ASSERT_EQ(PrintToString(e.info().msg), - PrintToString(hintfmt("Not much"))); + PrintToString(HintFmt("puppy"))); auto trace = e.info().traces.rbegin(); ASSERT_EQ(e.info().traces.size(), 2); ASSERT_EQ(PrintToString(trace->hint), - PrintToString(hintfmt("No more"))); + PrintToString(HintFmt("doggy"))); trace++; ASSERT_EQ(PrintToString(trace->hint), - PrintToString(hintfmt("Something"))); + PrintToString(HintFmt("beans"))); throw; } , EvalError @@ -47,12 +47,12 @@ namespace nix { TEST_F(ErrorTraceTest, NestedThrows) { try { - state.error("Not much").withTrace(noPos, "No more").debugThrow(); + state.error("puppy").withTrace(noPos, "doggy").debugThrow(); } catch (BaseError & e) { try { - state.error("Not much more").debugThrow(); + state.error("beans").debugThrow(); } catch (Error & e2) { - e.addTrace(state.positions[noPos], "Something", ""); + e.addTrace(state.positions[noPos], "beans2"); //e2.addTrace(state.positions[noPos], "Something", ""); ASSERT_TRUE(e.info().traces.size() == 2); ASSERT_TRUE(e2.info().traces.size() == 0); @@ -74,7 +74,7 @@ namespace nix { ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \ auto trace = e.info().traces.rbegin(); \ ASSERT_EQ(PrintToString(trace->hint), \ - PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ throw; \ } \ , type \ @@ -96,7 +96,7 @@ namespace nix { PrintToString(context)); \ ++trace; \ ASSERT_EQ(PrintToString(trace->hint), \ - PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ throw; \ } \ , type \ @@ -105,48 +105,48 @@ namespace nix { TEST_F(ErrorTraceTest, genericClosure) { ASSERT_TRACE2("genericClosure 1", TypeError, - hintfmt("value is %s while a set was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.genericClosure")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.genericClosure")); ASSERT_TRACE2("genericClosure {}", TypeError, - hintfmt("attribute '%s' missing", "startSet"), - hintfmt("in the attrset passed as argument to builtins.genericClosure")); + HintFmt("attribute '%s' missing", "startSet"), + HintFmt("in the attrset passed as argument to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = 1; }", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = true; }", TypeError, - hintfmt("value is %s while a function was expected", "a Boolean"), - hintfmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); + HintFmt("expected a function but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: true; }", TypeError, - hintfmt("value is %s while a list was expected", "a Boolean"), - hintfmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); + HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ true ]; }", TypeError, - hintfmt("value is %s while a set was expected", "a Boolean"), - hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ {} ]; }", TypeError, - hintfmt("attribute '%s' missing", "key"), - hintfmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); + HintFmt("attribute '%s' missing", "key"), + HintFmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [{ key = ''a''; }]; }", EvalError, - hintfmt("cannot compare %s with %s", "a string", "an integer"), - hintfmt("while comparing the `key` attributes of two genericClosure elements")); + HintFmt("cannot compare %s with %s", "a string", "an integer"), + HintFmt("while comparing the `key` attributes of two genericClosure elements")); ASSERT_TRACE2("genericClosure { startSet = [ true ]; operator = item: [{ key = ''a''; }]; }", TypeError, - hintfmt("value is %s while a set was expected", "a Boolean"), - hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); } @@ -154,32 +154,32 @@ namespace nix { TEST_F(ErrorTraceTest, replaceStrings) { ASSERT_TRACE2("replaceStrings 0 0 {}", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.replaceStrings")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); ASSERT_TRACE2("replaceStrings [] 0 {}", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the second argument passed to builtins.replaceStrings")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); ASSERT_TRACE1("replaceStrings [ 0 ] [] {}", EvalError, - hintfmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); + HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); ASSERT_TRACE2("replaceStrings [ 1 ] [ \"new\" ] {}", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", TypeError, - hintfmt("value is %s while a string was expected", "a Boolean"), - hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); + HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); ASSERT_TRACE2("replaceStrings [ \"old\" ] [ \"new\" ] {}", TypeError, - hintfmt("value is %s while a string was expected", "a set"), - hintfmt("while evaluating the third argument passed to builtins.replaceStrings")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); } @@ -243,8 +243,8 @@ namespace nix { TEST_F(ErrorTraceTest, ceil) { ASSERT_TRACE2("ceil \"foo\"", TypeError, - hintfmt("value is %s while a float was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.ceil")); + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.ceil")); } @@ -252,8 +252,8 @@ namespace nix { TEST_F(ErrorTraceTest, floor) { ASSERT_TRACE2("floor \"foo\"", TypeError, - hintfmt("value is %s while a float was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.floor")); + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.floor")); } @@ -265,8 +265,8 @@ namespace nix { TEST_F(ErrorTraceTest, getEnv) { ASSERT_TRACE2("getEnv [ ]", TypeError, - hintfmt("value is %s while a string was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.getEnv")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.getEnv")); } @@ -286,8 +286,8 @@ namespace nix { TEST_F(ErrorTraceTest, placeholder) { ASSERT_TRACE2("placeholder []", TypeError, - hintfmt("value is %s while a string was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.placeholder")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.placeholder")); } @@ -295,13 +295,13 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the first argument passed to builtins.toPath")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", EvalError, - hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the first argument passed to builtins.toPath")); + HintFmt("string '%s' doesn't represent an absolute path", "foo"), + HintFmt("while evaluating the first argument passed to builtins.toPath")); } @@ -309,8 +309,8 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string", "a Boolean"), - hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); + HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,13 +318,13 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while realising the context of a path")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", EvalError, - hintfmt("string '%s' doesn't represent an absolute path", "zorglub"), - hintfmt("while realising the context of a path")); + HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), + HintFmt("while realising the context of a path")); } @@ -332,8 +332,8 @@ namespace nix { TEST_F(ErrorTraceTest, baseNameOf) { ASSERT_TRACE2("baseNameOf []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); } @@ -377,30 +377,30 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, - hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); + HintFmt("string '%s' doesn't represent an absolute path", "foo"), + HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, - hintfmt("value is %s while a function was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.filterSource")); + HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.filterSource")); // Usupported by store "dummy" // ASSERT_TRACE2("filterSource (_: 1) ./.", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "an integer"), - // hintfmt("while adding path '/home/layus/projects/nix'")); + // HintFmt("attempt to call something which is not a function but %s", "an integer"), + // HintFmt("while adding path '/home/layus/projects/nix'")); // ASSERT_TRACE2("filterSource (_: _: 1) ./.", // TypeError, - // hintfmt("value is %s while a Boolean was expected", "an integer"), - // hintfmt("while evaluating the return value of the path filter function")); + // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), + // HintFmt("while evaluating the return value of the path filter function")); } @@ -412,8 +412,8 @@ namespace nix { TEST_F(ErrorTraceTest, attrNames) { ASSERT_TRACE2("attrNames []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the argument passed to builtins.attrNames")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the argument passed to builtins.attrNames")); } @@ -421,8 +421,8 @@ namespace nix { TEST_F(ErrorTraceTest, attrValues) { ASSERT_TRACE2("attrValues []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the argument passed to builtins.attrValues")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the argument passed to builtins.attrValues")); } @@ -430,18 +430,18 @@ namespace nix { TEST_F(ErrorTraceTest, getAttr) { ASSERT_TRACE2("getAttr [] []", TypeError, - hintfmt("value is %s while a string was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.getAttr")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.getAttr")); ASSERT_TRACE2("getAttr \"foo\" []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the second argument passed to builtins.getAttr")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.getAttr")); ASSERT_TRACE2("getAttr \"foo\" {}", TypeError, - hintfmt("attribute '%s' missing", "foo"), - hintfmt("in the attribute set under consideration")); + HintFmt("attribute '%s' missing", "foo"), + HintFmt("in the attribute set under consideration")); } @@ -453,13 +453,13 @@ namespace nix { TEST_F(ErrorTraceTest, hasAttr) { ASSERT_TRACE2("hasAttr [] []", TypeError, - hintfmt("value is %s while a string was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.hasAttr")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.hasAttr")); ASSERT_TRACE2("hasAttr \"foo\" []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the second argument passed to builtins.hasAttr")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.hasAttr")); } @@ -471,18 +471,18 @@ namespace nix { TEST_F(ErrorTraceTest, removeAttrs) { ASSERT_TRACE2("removeAttrs \"\" \"\"", TypeError, - hintfmt("value is %s while a set was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); ASSERT_TRACE2("removeAttrs \"\" [ 1 ]", TypeError, - hintfmt("value is %s while a set was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); ASSERT_TRACE2("removeAttrs \"\" [ \"1\" ]", TypeError, - hintfmt("value is %s while a set was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); } @@ -490,28 +490,28 @@ namespace nix { TEST_F(ErrorTraceTest, listToAttrs) { ASSERT_TRACE2("listToAttrs 1", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the argument passed to builtins.listToAttrs")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the argument passed to builtins.listToAttrs")); ASSERT_TRACE2("listToAttrs [ 1 ]", TypeError, - hintfmt("value is %s while a set was expected", "an integer"), - hintfmt("while evaluating an element of the list passed to builtins.listToAttrs")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); ASSERT_TRACE2("listToAttrs [ {} ]", TypeError, - hintfmt("attribute '%s' missing", "name"), - hintfmt("in a {name=...; value=...;} pair")); + HintFmt("attribute '%s' missing", "name"), + HintFmt("in a {name=...; value=...;} pair")); ASSERT_TRACE2("listToAttrs [ { name = 1; } ]", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); ASSERT_TRACE2("listToAttrs [ { name = \"foo\"; } ]", TypeError, - hintfmt("attribute '%s' missing", "value"), - hintfmt("in a {name=...; value=...;} pair")); + HintFmt("attribute '%s' missing", "value"), + HintFmt("in a {name=...; value=...;} pair")); } @@ -519,13 +519,13 @@ namespace nix { TEST_F(ErrorTraceTest, intersectAttrs) { ASSERT_TRACE2("intersectAttrs [] []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.intersectAttrs")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); ASSERT_TRACE2("intersectAttrs {} []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the second argument passed to builtins.intersectAttrs")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); } @@ -533,23 +533,23 @@ namespace nix { TEST_F(ErrorTraceTest, catAttrs) { ASSERT_TRACE2("catAttrs [] {}", TypeError, - hintfmt("value is %s while a string was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.catAttrs")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.catAttrs")); ASSERT_TRACE2("catAttrs \"foo\" {}", TypeError, - hintfmt("value is %s while a list was expected", "a set"), - hintfmt("while evaluating the second argument passed to builtins.catAttrs")); + HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.catAttrs")); ASSERT_TRACE2("catAttrs \"foo\" [ 1 ]", TypeError, - hintfmt("value is %s while a set was expected", "an integer"), - hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); ASSERT_TRACE2("catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", TypeError, - hintfmt("value is %s while a set was expected", "an integer"), - hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); } @@ -557,7 +557,7 @@ namespace nix { TEST_F(ErrorTraceTest, functionArgs) { ASSERT_TRACE1("functionArgs {}", TypeError, - hintfmt("'functionArgs' requires a function")); + HintFmt("'functionArgs' requires a function")); } @@ -565,24 +565,24 @@ namespace nix { TEST_F(ErrorTraceTest, mapAttrs) { ASSERT_TRACE2("mapAttrs [] []", TypeError, - hintfmt("value is %s while a set was expected", "a list"), - hintfmt("while evaluating the second argument passed to builtins.mapAttrs")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); // XXX: defered // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "a string"), - // hintfmt("while evaluating the attribute 'foo'")); + // HintFmt("attempt to call something which is not a function but %s", "a string"), + // HintFmt("while evaluating the attribute 'foo'")); // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "a string"), - // hintfmt("while evaluating the attribute 'foo'")); + // HintFmt("attempt to call something which is not a function but %s", "a string"), + // HintFmt("while evaluating the attribute 'foo'")); // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", // TypeError, - // hintfmt("cannot coerce %s to a string", "an integer"), - // hintfmt("while evaluating a path segment")); + // HintFmt("cannot coerce %s to a string", "an integer"), + // HintFmt("while evaluating a path segment")); } @@ -590,27 +590,27 @@ namespace nix { TEST_F(ErrorTraceTest, zipAttrsWith) { ASSERT_TRACE2("zipAttrsWith [] [ 1 ]", TypeError, - hintfmt("value is %s while a function was expected", "a list"), - hintfmt("while evaluating the first argument passed to builtins.zipAttrsWith")); + HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); ASSERT_TRACE2("zipAttrsWith (_: 1) [ 1 ]", TypeError, - hintfmt("value is %s while a set was expected", "an integer"), - hintfmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); // XXX: How to properly tell that the fucntion takes two arguments ? // The same question also applies to sort, and maybe others. // Due to lazyness, we only create a thunk, and it fails later on. // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "an integer"), - // hintfmt("while evaluating the attribute 'foo'")); + // HintFmt("attempt to call something which is not a function but %s", "an integer"), + // HintFmt("while evaluating the attribute 'foo'")); // XXX: Also deferred deeply // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", // TypeError, - // hintfmt("cannot coerce %s to a string", "a list"), - // hintfmt("while evaluating a path segment")); + // HintFmt("cannot coerce %s to a string", "a list"), + // HintFmt("while evaluating a path segment")); } @@ -622,16 +622,16 @@ namespace nix { TEST_F(ErrorTraceTest, elemAt) { ASSERT_TRACE2("elemAt \"foo\" (-1)", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.elemAt")); ASSERT_TRACE1("elemAt [] (-1)", Error, - hintfmt("list index %d is out of bounds", -1)); + HintFmt("list index %d is out of bounds", -1)); ASSERT_TRACE1("elemAt [\"foo\"] 3", Error, - hintfmt("list index %d is out of bounds", 3)); + HintFmt("list index %d is out of bounds", 3)); } @@ -639,12 +639,12 @@ namespace nix { TEST_F(ErrorTraceTest, head) { ASSERT_TRACE2("head 1", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.elemAt")); ASSERT_TRACE1("head []", Error, - hintfmt("list index %d is out of bounds", 0)); + HintFmt("list index %d is out of bounds", 0)); } @@ -652,12 +652,12 @@ namespace nix { TEST_F(ErrorTraceTest, tail) { ASSERT_TRACE2("tail 1", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.tail")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.tail")); ASSERT_TRACE1("tail []", Error, - hintfmt("'tail' called on an empty list")); + HintFmt("'tail' called on an empty list")); } @@ -665,13 +665,13 @@ namespace nix { TEST_F(ErrorTraceTest, map) { ASSERT_TRACE2("map 1 \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.map")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.map")); ASSERT_TRACE2("map 1 [ 1 ]", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.map")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.map")); } @@ -679,18 +679,18 @@ namespace nix { TEST_F(ErrorTraceTest, filter) { ASSERT_TRACE2("filter 1 \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.filter")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.filter")); ASSERT_TRACE2("filter 1 [ \"foo\" ]", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.filter")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.filter")); ASSERT_TRACE2("filter (_: 5) [ \"foo\" ]", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the return value of the filtering function passed to builtins.filter")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); } @@ -698,8 +698,8 @@ namespace nix { TEST_F(ErrorTraceTest, elem) { ASSERT_TRACE2("elem 1 \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.elem")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.elem")); } @@ -707,18 +707,18 @@ namespace nix { TEST_F(ErrorTraceTest, concatLists) { ASSERT_TRACE2("concatLists 1", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.concatLists")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.concatLists")); ASSERT_TRACE2("concatLists [ 1 ]", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed to builtins.concatLists")); ASSERT_TRACE2("concatLists [ [1] \"foo\" ]", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed to builtins.concatLists")); } @@ -726,13 +726,13 @@ namespace nix { TEST_F(ErrorTraceTest, length) { ASSERT_TRACE2("length 1", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.length")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.length")); ASSERT_TRACE2("length \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the first argument passed to builtins.length")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.length")); } @@ -740,22 +740,22 @@ namespace nix { TEST_F(ErrorTraceTest, foldlPrime) { ASSERT_TRACE2("foldl' 1 \"foo\" true", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.foldlStrict")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); ASSERT_TRACE2("foldl' (_: 1) \"foo\" true", TypeError, - hintfmt("value is %s while a list was expected", "a Boolean"), - hintfmt("while evaluating the third argument passed to builtins.foldlStrict")); + HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); ASSERT_TRACE1("foldl' (_: 1) \"foo\" [ true ]", TypeError, - hintfmt("attempt to call something which is not a function but %s", "an integer")); + HintFmt("attempt to call something which is not a function but %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); ASSERT_TRACE2("foldl' (a: b: a && b) \"foo\" [ true ]", TypeError, - hintfmt("value is %s while a Boolean was expected", "a string"), - hintfmt("in the left operand of the AND (&&) operator")); + HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("in the left operand of the AND (&&) operator")); } @@ -763,18 +763,18 @@ namespace nix { TEST_F(ErrorTraceTest, any) { ASSERT_TRACE2("any 1 \"foo\"", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.any")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.any")); ASSERT_TRACE2("any (_: 1) \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.any")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.any")); ASSERT_TRACE2("any (_: 1) [ \"foo\" ]", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to builtins.any")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.any")); } @@ -782,18 +782,18 @@ namespace nix { TEST_F(ErrorTraceTest, all) { ASSERT_TRACE2("all 1 \"foo\"", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.all")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.all")); ASSERT_TRACE2("all (_: 1) \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.all")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.all")); ASSERT_TRACE2("all (_: 1) [ \"foo\" ]", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to builtins.all")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.all")); } @@ -801,23 +801,23 @@ namespace nix { TEST_F(ErrorTraceTest, genList) { ASSERT_TRACE2("genList 1 \"foo\"", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.genList")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.genList")); ASSERT_TRACE2("genList 1 2", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.genList", "an integer")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.genList")); // XXX: defered // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", // TypeError, - // hintfmt("cannot add %s to an integer", "a string"), - // hintfmt("while evaluating anonymous lambda")); + // HintFmt("cannot add %s to an integer", "a string"), + // HintFmt("while evaluating anonymous lambda")); ASSERT_TRACE1("genList false (-3)", EvalError, - hintfmt("cannot create list of size %d", -3)); + HintFmt("cannot create list of size %d", -3)); } @@ -825,31 +825,31 @@ namespace nix { TEST_F(ErrorTraceTest, sort) { ASSERT_TRACE2("sort 1 \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.sort")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.sort")); ASSERT_TRACE2("sort 1 [ \"foo\" ]", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.sort")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.sort")); ASSERT_TRACE1("sort (_: 1) [ \"foo\" \"bar\" ]", TypeError, - hintfmt("attempt to call something which is not a function but %s", "an integer")); + HintFmt("attempt to call something which is not a function but %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); ASSERT_TRACE2("sort (_: _: 1) [ \"foo\" \"bar\" ]", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the return value of the sorting function passed to builtins.sort")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); // XXX: Trace too deep, need better asserts // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", // TypeError, - // hintfmt("cannot compare %s with %s", "a string", "a set")); + // HintFmt("cannot compare %s with %s", "a string", "a set")); // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", // TypeError, - // hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); } @@ -857,18 +857,18 @@ namespace nix { TEST_F(ErrorTraceTest, partition) { ASSERT_TRACE2("partition 1 \"foo\"", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.partition")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.partition")); ASSERT_TRACE2("partition (_: 1) \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.partition")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.partition")); ASSERT_TRACE2("partition (_: 1) [ \"foo\" ]", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the return value of the partition function passed to builtins.partition")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); } @@ -876,18 +876,18 @@ namespace nix { TEST_F(ErrorTraceTest, groupBy) { ASSERT_TRACE2("groupBy 1 \"foo\"", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.groupBy")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.groupBy")); ASSERT_TRACE2("groupBy (_: 1) \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.groupBy")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.groupBy")); ASSERT_TRACE2("groupBy (x: x) [ \"foo\" \"bar\" 1 ]", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); } @@ -895,23 +895,23 @@ namespace nix { TEST_F(ErrorTraceTest, concatMap) { ASSERT_TRACE2("concatMap 1 \"foo\"", TypeError, - hintfmt("value is %s while a function was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.concatMap")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: 1) \"foo\"", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the second argument passed to builtins.concatMap")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, - hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); } @@ -919,13 +919,13 @@ namespace nix { TEST_F(ErrorTraceTest, add) { ASSERT_TRACE2("add \"foo\" 1", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the first argument of the addition")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the addition")); ASSERT_TRACE2("add 1 \"foo\"", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the second argument of the addition")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the addition")); } @@ -933,13 +933,13 @@ namespace nix { TEST_F(ErrorTraceTest, sub) { ASSERT_TRACE2("sub \"foo\" 1", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the first argument of the subtraction")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the subtraction")); ASSERT_TRACE2("sub 1 \"foo\"", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the second argument of the subtraction")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the subtraction")); } @@ -947,13 +947,13 @@ namespace nix { TEST_F(ErrorTraceTest, mul) { ASSERT_TRACE2("mul \"foo\" 1", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the first argument of the multiplication")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the multiplication")); ASSERT_TRACE2("mul 1 \"foo\"", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the second argument of the multiplication")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the multiplication")); } @@ -961,17 +961,17 @@ namespace nix { TEST_F(ErrorTraceTest, div) { ASSERT_TRACE2("div \"foo\" 1 # TODO: an integer was expected -> a number", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the first operand of the division")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first operand of the division")); ASSERT_TRACE2("div 1 \"foo\"", TypeError, - hintfmt("value is %s while a float was expected", "a string"), - hintfmt("while evaluating the second operand of the division")); + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second operand of the division")); ASSERT_TRACE1("div \"foo\" 0", EvalError, - hintfmt("division by zero")); + HintFmt("division by zero")); } @@ -979,13 +979,13 @@ namespace nix { TEST_F(ErrorTraceTest, bitAnd) { ASSERT_TRACE2("bitAnd 1.1 2", TypeError, - hintfmt("value is %s while an integer was expected", "a float"), - hintfmt("while evaluating the first argument passed to builtins.bitAnd")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitAnd")); ASSERT_TRACE2("bitAnd 1 2.2", TypeError, - hintfmt("value is %s while an integer was expected", "a float"), - hintfmt("while evaluating the second argument passed to builtins.bitAnd")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitAnd")); } @@ -993,13 +993,13 @@ namespace nix { TEST_F(ErrorTraceTest, bitOr) { ASSERT_TRACE2("bitOr 1.1 2", TypeError, - hintfmt("value is %s while an integer was expected", "a float"), - hintfmt("while evaluating the first argument passed to builtins.bitOr")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitOr")); ASSERT_TRACE2("bitOr 1 2.2", TypeError, - hintfmt("value is %s while an integer was expected", "a float"), - hintfmt("while evaluating the second argument passed to builtins.bitOr")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitOr")); } @@ -1007,13 +1007,13 @@ namespace nix { TEST_F(ErrorTraceTest, bitXor) { ASSERT_TRACE2("bitXor 1.1 2", TypeError, - hintfmt("value is %s while an integer was expected", "a float"), - hintfmt("while evaluating the first argument passed to builtins.bitXor")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitXor")); ASSERT_TRACE2("bitXor 1 2.2", TypeError, - hintfmt("value is %s while an integer was expected", "a float"), - hintfmt("while evaluating the second argument passed to builtins.bitXor")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitXor")); } @@ -1021,16 +1021,16 @@ namespace nix { TEST_F(ErrorTraceTest, lessThan) { ASSERT_TRACE1("lessThan 1 \"foo\"", EvalError, - hintfmt("cannot compare %s with %s", "an integer", "a string")); + HintFmt("cannot compare %s with %s", "an integer", "a string")); ASSERT_TRACE1("lessThan {} {}", EvalError, - hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); ASSERT_TRACE2("lessThan [ 1 2 ] [ \"foo\" ]", EvalError, - hintfmt("cannot compare %s with %s", "an integer", "a string"), - hintfmt("while comparing two list elements")); + HintFmt("cannot compare %s with %s", "an integer", "a string"), + HintFmt("while comparing two list elements")); } @@ -1038,8 +1038,8 @@ namespace nix { TEST_F(ErrorTraceTest, toString) { ASSERT_TRACE2("toString { a = 1; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating the first argument passed to builtins.toString")); + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), + HintFmt("while evaluating the first argument passed to builtins.toString")); } @@ -1047,22 +1047,22 @@ namespace nix { TEST_F(ErrorTraceTest, substring) { ASSERT_TRACE2("substring {} \"foo\" true", TypeError, - hintfmt("value is %s while an integer was expected", "a set"), - hintfmt("while evaluating the first argument (the start offset) passed to builtins.substring")); + HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); ASSERT_TRACE2("substring 3 \"foo\" true", TypeError, - hintfmt("value is %s while an integer was expected", "a string"), - hintfmt("while evaluating the second argument (the substring length) passed to builtins.substring")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); ASSERT_TRACE2("substring 0 3 {}", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, - hintfmt("negative start position in 'substring'")); + HintFmt("negative start position in 'substring'")); } @@ -1070,8 +1070,8 @@ namespace nix { TEST_F(ErrorTraceTest, stringLength) { ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating the argument passed to builtins.stringLength")); + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the argument passed to builtins.stringLength")); } @@ -1079,17 +1079,17 @@ namespace nix { TEST_F(ErrorTraceTest, hashString) { ASSERT_TRACE2("hashString 1 {}", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.hashString")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.hashString")); ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); + HintFmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, - hintfmt("value is %s while a string was expected", "a set"), - hintfmt("while evaluating the second argument passed to builtins.hashString")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.hashString")); } @@ -1097,17 +1097,17 @@ namespace nix { TEST_F(ErrorTraceTest, match) { ASSERT_TRACE2("match 1 {}", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.match")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.match")); ASSERT_TRACE2("match \"foo\" {}", TypeError, - hintfmt("value is %s while a string was expected", "a set"), - hintfmt("while evaluating the second argument passed to builtins.match")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.match")); ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, - hintfmt("invalid regular expression '%s'", "(.*")); + HintFmt("invalid regular expression '%s'", "(.*")); } @@ -1115,17 +1115,17 @@ namespace nix { TEST_F(ErrorTraceTest, split) { ASSERT_TRACE2("split 1 {}", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.split")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.split")); ASSERT_TRACE2("split \"foo\" {}", TypeError, - hintfmt("value is %s while a string was expected", "a set"), - hintfmt("while evaluating the second argument passed to builtins.split")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.split")); ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, - hintfmt("invalid regular expression '%s'", "f(o*o")); + HintFmt("invalid regular expression '%s'", "f(o*o")); } @@ -1133,18 +1133,18 @@ namespace nix { TEST_F(ErrorTraceTest, concatStringsSep) { ASSERT_TRACE2("concatStringsSep 1 {}", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); ASSERT_TRACE2("concatStringsSep \"foo\" {}", TypeError, - hintfmt("value is %s while a list was expected", "a set"), - hintfmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); + HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", TypeError, - hintfmt("cannot coerce %s to a string", "an integer"), - hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); + HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); } @@ -1152,8 +1152,8 @@ namespace nix { TEST_F(ErrorTraceTest, parseDrvName) { ASSERT_TRACE2("parseDrvName 1", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.parseDrvName")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); } @@ -1161,13 +1161,13 @@ namespace nix { TEST_F(ErrorTraceTest, compareVersions) { ASSERT_TRACE2("compareVersions 1 {}", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.compareVersions")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.compareVersions")); ASSERT_TRACE2("compareVersions \"abd\" {}", TypeError, - hintfmt("value is %s while a string was expected", "a set"), - hintfmt("while evaluating the second argument passed to builtins.compareVersions")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.compareVersions")); } @@ -1175,8 +1175,8 @@ namespace nix { TEST_F(ErrorTraceTest, splitVersion) { ASSERT_TRACE2("splitVersion 1", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the first argument passed to builtins.splitVersion")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.splitVersion")); } @@ -1189,108 +1189,108 @@ namespace nix { TEST_F(ErrorTraceTest, derivationStrict) { ASSERT_TRACE2("derivationStrict \"\"", TypeError, - hintfmt("value is %s while a set was expected", "a string"), - hintfmt("while evaluating the argument passed to builtins.derivationStrict")); + HintFmt("expected a set but found %s: %s", "a string", "\"\""), + HintFmt("while evaluating the argument passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict {}", TypeError, - hintfmt("attribute '%s' missing", "name"), - hintfmt("in the attrset passed as argument to builtins.derivationStrict")); + HintFmt("attribute '%s' missing", "name"), + HintFmt("in the attrset passed as argument to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = 1; }", TypeError, - hintfmt("value is %s while a string was expected", "an integer"), - hintfmt("while evaluating the `name` attribute passed to builtins.derivationStrict")); + HintFmt("expected a string but found %s: %s", "an integer", "1"), + HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; }", TypeError, - hintfmt("required attribute 'builder' missing"), - hintfmt("while evaluating derivation 'foo'")); + HintFmt("required attribute 'builder' missing"), + HintFmt("while evaluating derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); + HintFmt("expected a Boolean but found %s: %s", "an integer", "15"), + HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", TypeError, - hintfmt("value is %s while a Boolean was expected", "an integer"), - hintfmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); + HintFmt("expected a Boolean but found %s: %s", "an integer", "15"), + HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", TypeError, - hintfmt("invalid value '15' for 'outputHashMode' attribute"), - hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + HintFmt("invalid value '15' for 'outputHashMode' attribute"), + HintFmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", TypeError, - hintfmt("invalid value 'custom' for 'outputHashMode' attribute"), - hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + HintFmt("invalid value 'custom' for 'outputHashMode' attribute"), + HintFmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating the attribute 'system' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", TypeError, - hintfmt("invalid derivation output name 'drv'"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("invalid derivation output name 'drv'"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", TypeError, - hintfmt("derivation cannot have an empty set of outputs"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("derivation cannot have an empty set of outputs"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drv\" ]; }", TypeError, - hintfmt("invalid derivation output name 'drv'"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("invalid derivation output name 'drv'"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", TypeError, - hintfmt("duplicate derivation output 'out'"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("duplicate derivation output 'out'"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", TypeError, - hintfmt("value is %s while a Boolean was expected", "a string"), - hintfmt("while evaluating the attribute '__contentAddressed' of derivation 'foo'")); + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt("while evaluating the attribute '__contentAddressed' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", TypeError, - hintfmt("value is %s while a Boolean was expected", "a string"), - hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt("while evaluating the attribute '__impure' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", TypeError, - hintfmt("value is %s while a Boolean was expected", "a string"), - hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt("while evaluating the attribute '__impure' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", TypeError, - hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the attribute 'args' of derivation 'foo'")); + HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), + HintFmt("while evaluating the attribute 'args' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating an element of the argument list")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating an element of the argument list")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), - hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating the attribute 'FOO' of derivation 'foo'")); } */ diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc new file mode 100644 index 00000000000..93d3f658f52 --- /dev/null +++ b/tests/unit/libexpr/eval.cc @@ -0,0 +1,141 @@ +#include +#include + +#include "eval.hh" +#include "tests/libexpr.hh" + +namespace nix { + +TEST(nix_isAllowedURI, http_example_com) { + Strings allowed; + allowed.push_back("http://example.com"); + + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.co", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); +} + +TEST(nix_isAllowedURI, http_example_com_foo) { + Strings allowed; + allowed.push_back("http://example.com/foo"); + + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); + // Broken? + // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed)); +} + +TEST(nix_isAllowedURI, http) { + Strings allowed; + allowed.push_back("http://"); + + ASSERT_TRUE(isAllowedURI("http://", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("https://", allowed)); + ASSERT_FALSE(isAllowedURI("http:foo", allowed)); +} + +TEST(nix_isAllowedURI, https) { + Strings allowed; + allowed.push_back("https://"); + + ASSERT_TRUE(isAllowedURI("https://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed)); +} + +TEST(nix_isAllowedURI, absolute_path) { + Strings allowed; + allowed.push_back("/var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("/var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); +} + +TEST(nix_isAllowedURI, file_url) { + Strings allowed; + allowed.push_back("file:///var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///", allowed)); + ASSERT_FALSE(isAllowedURI("file://", allowed)); +} + +TEST(nix_isAllowedURI, github_all) { + Strings allowed; + allowed.push_back("github:"); + ASSERT_TRUE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("github", allowed)); +} + +TEST(nix_isAllowedURI, github_org) { + Strings allowed; + allowed.push_back("github:foo"); + ASSERT_FALSE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); +} + +TEST(nix_isAllowedURI, non_scheme_colon) { + Strings allowed; + allowed.push_back("https://foo/bar:"); + ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed)); + ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed)); + ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); +} + +} // namespace nix \ No newline at end of file diff --git a/tests/unit/libexpr/flakeref.cc b/tests/unit/libexpr/flake/flakeref.cc similarity index 100% rename from tests/unit/libexpr/flakeref.cc rename to tests/unit/libexpr/flake/flakeref.cc diff --git a/tests/unit/libexpr/flake/url-name.cc b/tests/unit/libexpr/flake/url-name.cc new file mode 100644 index 00000000000..85387b323e3 --- /dev/null +++ b/tests/unit/libexpr/flake/url-name.cc @@ -0,0 +1,69 @@ +#include "flake/url-name.hh" +#include + +namespace nix { + +/* ----------- tests for url-name.hh --------------------------------------------------*/ + + TEST(getNameFromURL, getNameFromURL) { + ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#legacyPackages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex"); + ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj"); + + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + ASSERT_EQ(getNameFromURL(parseURL("github:edolstra/nix-warez?rev=1234&dir=blender&ref=master")), "blender"); + + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/dwarffs")), "dwarffs"); + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/nix-warez?dir=blender")), "blender"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh://git@github.com/someuser/my-repo#")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+git://github.com/someuser/my-repo?rev=v1.2.3")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh:///home/user/project?dir=subproject&rev=v2.4")), "subproject"); + ASSERT_EQ(getNameFromURL(parseURL("git+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("git+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("tarball+http://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.jq")), "jq"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+https://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.hg")), "hg"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+file:///home/user/Downloads/nixpkgs-2.18.1#packages.aarch64-darwin.ripgrep")), "ripgrep"); + + ASSERT_EQ(getNameFromURL(parseURL("https://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + ASSERT_EQ(getNameFromURL(parseURL("http://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + + ASSERT_EQ(getNameFromURL(parseURL("file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://gitfantasy.com/org/user/notaflake")), "notaflake"); + ASSERT_EQ(getNameFromURL(parseURL("file+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("https://www.github.com/")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt); + } +} diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index 5743880d7d9..25810ad9cf8 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -16,7 +16,8 @@ endif libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) libexpr-tests_EXTRA_INCLUDES = \ -I tests/unit/libexpr-support \ diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 384d9924b33..6d7649b3c00 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -1,6 +1,7 @@ #include #include +#include "eval-settings.hh" #include "memory-input-accessor.hh" #include "tests/libexpr.hh" @@ -603,7 +604,7 @@ namespace nix { ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1")); } - TEST_F(PrimOpTest, hashStringInvalidHashType) { + TEST_F(PrimOpTest, hashStringInvalidHashAlgorithm) { ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error); } @@ -631,7 +632,7 @@ namespace nix { TEST_F(PrimOpTest, currentSystem) { auto v = eval("builtins.currentSystem"); - ASSERT_THAT(v, IsStringEq(settings.thisSystem.get())); + ASSERT_THAT(v, IsStringEq(evalSettings.getCurrentSystem())); } TEST_F(PrimOpTest, derivation) { diff --git a/tests/unit/libexpr/value/print.cc b/tests/unit/libexpr/value/print.cc index a4f6fc014f3..aabf156c29e 100644 --- a/tests/unit/libexpr/value/print.cc +++ b/tests/unit/libexpr/value/print.cc @@ -1,6 +1,7 @@ #include "tests/libexpr.hh" #include "value.hh" +#include "print.hh" namespace nix { @@ -12,7 +13,7 @@ struct ValuePrintingTests : LibExprTest void test(Value v, std::string_view expected, A... args) { std::stringstream out; - v.print(state.symbols, out, args...); + v.print(state, out, args...); ASSERT_EQ(out.str(), expected); } }; @@ -84,7 +85,7 @@ TEST_F(ValuePrintingTests, tList) vList.bigList.elems[1] = &vTwo; vList.bigList.size = 3; - test(vList, "[ 1 2 (nullptr) ]"); + test(vList, "[ 1 2 «nullptr» ]"); } TEST_F(ValuePrintingTests, vThunk) @@ -92,7 +93,7 @@ TEST_F(ValuePrintingTests, vThunk) Value vThunk; vThunk.mkThunk(nullptr, nullptr); - test(vThunk, ""); + test(vThunk, "«thunk»"); } TEST_F(ValuePrintingTests, vApp) @@ -100,32 +101,55 @@ TEST_F(ValuePrintingTests, vApp) Value vApp; vApp.mkApp(nullptr, nullptr); - test(vApp, ""); + test(vApp, "«thunk»"); } TEST_F(ValuePrintingTests, vLambda) { + Env env { + .up = nullptr, + .values = { } + }; + PosTable::Origin origin((std::monostate())); + auto posIdx = state.positions.add(origin, 1, 1); + auto body = ExprInt(0); + auto formals = Formals {}; + + ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body); + Value vLambda; - vLambda.mkLambda(nullptr, nullptr); + vLambda.mkLambda(&env, &eLambda); + + test(vLambda, "«lambda @ «none»:1:1»"); + + eLambda.setName(createSymbol("puppy")); - test(vLambda, ""); + test(vLambda, "«lambda puppy @ «none»:1:1»"); } TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - PrimOp primOp{}; + PrimOp primOp{ + .name = "puppy" + }; vPrimOp.mkPrimOp(&primOp); - test(vPrimOp, ""); + test(vPrimOp, "«primop puppy»"); } TEST_F(ValuePrintingTests, vPrimOpApp) { + PrimOp primOp{ + .name = "puppy" + }; + Value vPrimOp; + vPrimOp.mkPrimOp(&primOp); + Value vPrimOpApp; - vPrimOpApp.mkPrimOpApp(nullptr, nullptr); + vPrimOpApp.mkPrimOpApp(&vPrimOp, nullptr); - test(vPrimOpApp, ""); + test(vPrimOpApp, "«partially applied primop puppy»"); } TEST_F(ValuePrintingTests, vExternal) @@ -176,9 +200,14 @@ TEST_F(ValuePrintingTests, depthAttrs) Value vTwo; vTwo.mkInt(2); + BindingsBuilder builderEmpty(state, state.allocBindings(0)); + Value vAttrsEmpty; + vAttrsEmpty.mkAttrs(builderEmpty.finish()); + BindingsBuilder builder(state, state.allocBindings(10)); builder.insert(state.symbols.create("one"), &vOne); builder.insert(state.symbols.create("two"), &vTwo); + builder.insert(state.symbols.create("nested"), &vAttrsEmpty); Value vAttrs; vAttrs.mkAttrs(builder.finish()); @@ -191,10 +220,10 @@ TEST_F(ValuePrintingTests, depthAttrs) Value vNested; vNested.mkAttrs(builder2.finish()); - test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1); - test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2); - test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3); - test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4); + test(vNested, "{ nested = { ... }; one = 1; two = 2; }", PrintOptions { .maxDepth = 1 }); + test(vNested, "{ nested = { nested = { ... }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 2 }); + test(vNested, "{ nested = { nested = { }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 3 }); + test(vNested, "{ nested = { nested = { }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 4 }); } TEST_F(ValuePrintingTests, depthList) @@ -227,11 +256,525 @@ TEST_F(ValuePrintingTests, depthList) vList.bigList.elems[2] = &vNested; vList.bigList.size = 3; - test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1); - test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2); - test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3); - test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4); - test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5); + test(vList, "[ 1 2 { ... } ]", PrintOptions { .maxDepth = 1 }); + test(vList, "[ 1 2 { nested = { ... }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 2 }); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 3 }); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 4 }); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 5 }); +} + +struct StringPrintingTests : LibExprTest +{ + template + void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args) + { + Value v; + v.mkString(literal); + + std::stringstream out; + printValue(state, out, v, PrintOptions { + .maxStringLength = maxLength + }); + ASSERT_EQ(out.str(), expected); + } +}; + +TEST_F(StringPrintingTests, maxLengthTruncation) +{ + test("abcdefghi", "\"abcdefghi\"", 10); + test("abcdefghij", "\"abcdefghij\"", 10); + test("abcdefghijk", "\"abcdefghij\" «1 byte elided»", 10); + test("abcdefghijkl", "\"abcdefghij\" «2 bytes elided»", 10); + test("abcdefghijklm", "\"abcdefghij\" «3 bytes elided»", 10); +} + +// Check that printing an attrset shows 'important' attributes like `type` +// first, but only reorder the attrs when we have a maxAttrs budget. +TEST_F(ValuePrintingTests, attrsTypeFirst) +{ + Value vType; + vType.mkString("puppy"); + + Value vApple; + vApple.mkString("apple"); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("type"), &vType); + builder.insert(state.symbols.create("apple"), &vApple); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ type = \"puppy\"; apple = \"apple\"; }", + PrintOptions { + .maxAttrs = 100 + }); + + test(vAttrs, + "{ apple = \"apple\"; type = \"puppy\"; }", + PrintOptions { }); +} + +TEST_F(ValuePrintingTests, ansiColorsInt) +{ + Value v; + v.mkInt(10); + + test(v, + ANSI_CYAN "10" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsFloat) +{ + Value v; + v.mkFloat(1.6); + + test(v, + ANSI_CYAN "1.6" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsBool) +{ + Value v; + v.mkBool(true); + + test(v, + ANSI_CYAN "true" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsString) +{ + Value v; + v.mkString("puppy"); + + test(v, + ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsStringElided) +{ + Value v; + v.mkString("puppy"); + + test(v, + ANSI_MAGENTA "\"pup\" " ANSI_FAINT "«2 bytes elided»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .maxStringLength = 3 + }); +} + +TEST_F(ValuePrintingTests, ansiColorsPath) +{ + Value v; + v.mkPath(state.rootPath(CanonPath("puppy"))); + + test(v, + ANSI_GREEN "/puppy" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsNull) +{ + Value v; + v.mkNull(); + + test(v, + ANSI_CYAN "null" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; two = " ANSI_CYAN "2" ANSI_NORMAL "; }", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsDerivation) +{ + Value vDerivation; + vDerivation.mkString("derivation"); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.sType, &vDerivation); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + ANSI_GREEN "«derivation»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true, + .derivationPaths = true + }); + + test(vAttrs, + "{ type = " ANSI_MAGENTA "\"derivation\"" ANSI_NORMAL "; }", + PrintOptions { + .ansiColors = true, + .force = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsError) +{ + Value throw_ = state.getBuiltin("throw"); + Value message; + message.mkString("uh oh!"); + Value vError; + vError.mkApp(&throw_, &message); + + test(vError, + ANSI_RED + "«error: uh oh!»" + ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true, + }); +} + +TEST_F(ValuePrintingTests, ansiColorsDerivationError) +{ + Value throw_ = state.getBuiltin("throw"); + Value message; + message.mkString("uh oh!"); + Value vError; + vError.mkApp(&throw_, &message); + + Value vDerivation; + vDerivation.mkString("derivation"); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.sType, &vDerivation); + builder.insert(state.sDrvPath, &vError); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ drvPath = " + ANSI_RED + "«error: uh oh!»" + ANSI_NORMAL + "; type = " + ANSI_MAGENTA + "\"derivation\"" + ANSI_NORMAL + "; }", + PrintOptions { + .ansiColors = true, + .force = true + }); + + test(vAttrs, + ANSI_RED + "«error: uh oh!»" + ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true, + .derivationPaths = true, + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAssert) +{ + ExprVar eFalse(state.symbols.create("false")); + eFalse.bindVars(state, state.staticBaseEnv); + ExprInt eInt(1); + + ExprAssert expr(noPos, &eFalse, &eInt); + + Value v; + state.mkThunk_(v, &expr); + + test(v, + ANSI_RED "«error: assertion 'false' failed»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.size = 3; + + test(vList, + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_CYAN "2" ANSI_NORMAL " " ANSI_MAGENTA "«nullptr»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsLambda) +{ + Env env { + .up = nullptr, + .values = { } + }; + PosTable::Origin origin((std::monostate())); + auto posIdx = state.positions.add(origin, 1, 1); + auto body = ExprInt(0); + auto formals = Formals {}; + + ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body); + + Value vLambda; + vLambda.mkLambda(&env, &eLambda); + + test(vLambda, + ANSI_BLUE "«lambda @ «none»:1:1»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true + }); + + eLambda.setName(createSymbol("puppy")); + + test(vLambda, + ANSI_BLUE "«lambda puppy @ «none»:1:1»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsPrimOp) +{ + PrimOp primOp{ + .name = "puppy" + }; + Value v; + v.mkPrimOp(&primOp); + + test(v, + ANSI_BLUE "«primop puppy»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsPrimOpApp) +{ + PrimOp primOp{ + .name = "puppy" + }; + Value vPrimOp; + vPrimOp.mkPrimOp(&primOp); + + Value v; + v.mkPrimOpApp(&vPrimOp, nullptr); + + test(v, + ANSI_BLUE "«partially applied primop puppy»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsThunk) +{ + Value v; + v.mkThunk(nullptr, nullptr); + + test(v, + ANSI_MAGENTA "«thunk»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsBlackhole) +{ + Value v; + v.mkBlackhole(); + + test(v, + ANSI_RED "«potential infinite recursion»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vEmpty; + vEmpty.mkAttrs(emptyBuilder.finish()); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("a"), &vEmpty); + builder.insert(state.symbols.create("b"), &vEmpty); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ a = { }; b = " ANSI_MAGENTA "«repeated»" ANSI_NORMAL "; }", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsListRepeated) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vEmpty; + vEmpty.mkAttrs(emptyBuilder.finish()); + + Value vList; + state.mkList(vList, 3); + vList.bigList.elems[0] = &vEmpty; + vList.bigList.elems[1] = &vEmpty; + vList.bigList.size = 2; + + test(vList, + "[ { } " ANSI_MAGENTA "«repeated»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, listRepeated) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vEmpty; + vEmpty.mkAttrs(emptyBuilder.finish()); + + Value vList; + state.mkList(vList, 3); + vList.bigList.elems[0] = &vEmpty; + vList.bigList.elems[1] = &vEmpty; + vList.bigList.size = 2; + + test(vList, "[ { } «repeated» ]", PrintOptions { }); + test(vList, + "[ { } { } ]", + PrintOptions { + .trackRepeated = false + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAttrsElided) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«1 attribute elided»" ANSI_NORMAL " }", + PrintOptions { + .ansiColors = true, + .maxAttrs = 1 + }); + + Value vThree; + vThree.mkInt(3); + + builder.insert(state.symbols.create("three"), &vThree); + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }", + PrintOptions { + .ansiColors = true, + .maxAttrs = 1 + }); +} + +TEST_F(ValuePrintingTests, ansiColorsListElided) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + Value vList; + state.mkList(vList, 4); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.size = 2; + + test(vList, + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«1 item elided»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true, + .maxListItems = 1 + }); + + Value vThree; + vThree.mkInt(3); + + vList.bigList.elems[2] = &vThree; + vList.bigList.size = 3; + + test(vList, + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«2 items elided»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true, + .maxListItems = 1 + }); } } // namespace nix diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk index ff075c96a69..56dedd825d8 100644 --- a/tests/unit/libstore-support/local.mk +++ b/tests/unit/libstore-support/local.mk @@ -18,4 +18,4 @@ libstore-test-support_LIBS = \ libutil-test-support \ libstore libutil -libstore-test-support_LDFLAGS := -pthread -lrapidcheck +libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc index e5f169e94c5..8ddda80277c 100644 --- a/tests/unit/libstore-support/tests/path.cc +++ b/tests/unit/libstore-support/tests/path.cc @@ -1,3 +1,4 @@ +#include #include #include @@ -20,63 +21,60 @@ void showValue(const StorePath & p, std::ostream & os) namespace rc { using namespace nix; -Gen Arbitrary::arbitrary() +Gen storePathChar() { - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - StorePath::HashLen); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + return rc::gen::apply([](uint8_t i) -> char { + switch (i) { case 0 ... 9: - pre += '0' + i; + return '0' + i; case 10 ... 35: - pre += 'A' + (i - 10); - break; + return 'A' + (i - 10); case 36 ... 61: - pre += 'a' + (i - 36); - break; + return 'a' + (i - 36); case 62: - pre += '+'; - break; + return '+'; case 63: - pre += '-'; - break; + return '-'; case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } + return '.'; case 65: - pre += '_'; - break; + return '_'; case 66: - pre += '?'; - break; + return '?'; case 67: - pre += '='; - break; + return '='; default: assert(false); } - } + }, + gen::inRange(0, 10 + 2 * 26 + 6)); +} - return gen::just(StorePathName { - .name = std::move(pre), - }); +Gen Arbitrary::arbitrary() +{ + return gen::construct( + gen::suchThat( + gen::container(storePathChar()), + [](const std::string & s) { + return + !( s == "" + || s == "." + || s == ".." + || s.starts_with(".-") + || s.starts_with("..-") + ); + } + ) + ); } Gen Arbitrary::arbitrary() { - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); + return + gen::construct( + gen::arbitrary(), + gen::apply([](StorePathName n){ return n.name; }, gen::arbitrary()) + ); } } // namespace rc diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc new file mode 100644 index 00000000000..98c1eace368 --- /dev/null +++ b/tests/unit/libstore/content-address.cc @@ -0,0 +1,35 @@ +#include + +#include "content-address.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * ContentAddressMethod::parse, ContentAddressMethod::render + * --------------------------------------------------------------------------*/ + +TEST(ContentAddressMethod, testRoundTripPrintParse_1) { + for (const ContentAddressMethod & cam : { + ContentAddressMethod { TextIngestionMethod {} }, + ContentAddressMethod { FileIngestionMethod::Flat }, + ContentAddressMethod { FileIngestionMethod::Recursive }, + }) { + EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); + } +} + +TEST(ContentAddressMethod, testRoundTripPrintParse_2) { + for (const std::string_view camS : { + "text", + "flat", + "nar", + }) { + EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); + } +} + +TEST(ContentAddressMethod, testParseContentAddressMethodOptException) { + EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError); +} + +} diff --git a/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin b/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin new file mode 100644 index 00000000000..15ba4b5e3d9 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin new file mode 100644 index 00000000000..7d6e43fff2f Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin differ diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 5b66e5a5be9..9fd7fda54cc 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -14,6 +14,7 @@ using testing::SizeIs; using nix::absPath; using nix::FormatError; +using nix::UsageError; using nix::getMachines; using nix::Machine; using nix::Machines; @@ -133,7 +134,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; EXPECT_THROW(getMachines(), FormatError); settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; - EXPECT_THROW(getMachines(), FormatError); + EXPECT_THROW(getMachines(), UsageError); settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; EXPECT_THROW(getMachines(), FormatError); } diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc index 30631b5fd65..213b6e95f52 100644 --- a/tests/unit/libstore/path.cc +++ b/tests/unit/libstore/path.cc @@ -39,7 +39,12 @@ TEST_DONT_PARSE(double_star, "**") TEST_DONT_PARSE(star_first, "*,foo") TEST_DONT_PARSE(star_second, "foo,*") TEST_DONT_PARSE(bang, "foo!o") -TEST_DONT_PARSE(dotfile, ".gitignore") +TEST_DONT_PARSE(dot, ".") +TEST_DONT_PARSE(dot_dot, "..") +TEST_DONT_PARSE(dot_dot_dash, "..-1") +TEST_DONT_PARSE(dot_dash, ".-1") +TEST_DONT_PARSE(dot_dot_dash_a, "..-a") +TEST_DONT_PARSE(dot_dash_a, ".-a") #undef TEST_DONT_PARSE @@ -63,6 +68,11 @@ TEST_DO_PARSE(underscore, "foo_bar") TEST_DO_PARSE(period, "foo.txt") TEST_DO_PARSE(question_mark, "foo?why") TEST_DO_PARSE(equals_sign, "foo=foo") +TEST_DO_PARSE(dotfile, ".gitignore") +TEST_DO_PARSE(triple_dot_a, "...a") +TEST_DO_PARSE(triple_dot_1, "...1") +TEST_DO_PARSE(triple_dot_dash, "...-") +TEST_DO_PARSE(triple_dot, "...") #undef TEST_DO_PARSE @@ -84,6 +94,64 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); } + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_check_regex_eq_parse, + ()) +{ + static auto nameFuzzer = + rc::gen::container( + rc::gen::oneOf( + // alphanum, repeated to weigh heavier + rc::gen::oneOf( + rc::gen::inRange('0', '9'), + rc::gen::inRange('a', 'z'), + rc::gen::inRange('A', 'Z') + ), + // valid symbols + rc::gen::oneOf( + rc::gen::just('+'), + rc::gen::just('-'), + rc::gen::just('.'), + rc::gen::just('_'), + rc::gen::just('?'), + rc::gen::just('=') + ), + // symbols for scary .- and ..- cases, repeated for weight + rc::gen::just('.'), rc::gen::just('.'), + rc::gen::just('.'), rc::gen::just('.'), + rc::gen::just('-'), rc::gen::just('-'), + // ascii symbol ranges + rc::gen::oneOf( + rc::gen::inRange(' ', '/'), + rc::gen::inRange(':', '@'), + rc::gen::inRange('[', '`'), + rc::gen::inRange('{', '~') + ), + // typical whitespace + rc::gen::oneOf( + rc::gen::just(' '), + rc::gen::just('\t'), + rc::gen::just('\n'), + rc::gen::just('\r') + ), + // some chance of control codes, non-ascii or other garbage we missed + rc::gen::inRange('\0', '\xff') + )); + + auto name = *nameFuzzer; + + std::string path = store->storeDir + "/575s52sh487i0ylmbs9pvi606ljdszr0-" + name; + bool parsed = false; + try { + store->parseStorePath(path); + parsed = true; + } catch (const BadStorePath &) { + } + RC_ASSERT(parsed == std::regex_match(std::string { name }, nameRegex)); +} + #endif } diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index 8f256d1e632..b2fd0fb8248 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -1,3 +1,4 @@ +#include #include #include @@ -6,6 +7,7 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "build-result.hh" +#include "file-descriptor.hh" #include "tests/protocol.hh" #include "tests/characterization.hh" @@ -401,4 +403,112 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) +TEST_F(ServeProtoTest, handshake_log) +{ + CharacterizationTest::writeTest("handshake-to-client", [&]() -> std::string { + StringSink toClientLog; + + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + ServeProto::Version clientResult; + + auto thread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in0 { toClient.readSide.get() }; + TeeSource in { in0, toClientLog }; + clientResult = ServeProto::BasicClientConnection::handshake( + out, in, defaultVersion, "blah"); + }); + + { + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + ServeProto::BasicServerConnection::handshake( + out, in, defaultVersion); + }; + + thread.join(); + + return std::move(toClientLog.s); + }); +} + +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink { + void writeUnbuffered(std::string_view data) override { } +}; + +TEST_F(ServeProtoTest, handshake_client_replay) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + NullBufferedSink nullSink; + + StringSource in { toClientLog }; + auto clientResult = ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"); + + EXPECT_EQ(clientResult, defaultVersion); + }); +} + +TEST_F(ServeProtoTest, handshake_client_truncated_replay_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + for (size_t len = 0; len < toClientLog.size(); ++len) { + NullBufferedSink nullSink; + StringSource in { + // truncate + toClientLog.substr(0, len) + }; + if (len < 8) { + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + EndOfFile); + } else { + // Not sure why cannot keep on checking for `EndOfFile`. + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + Error); + } + } + }); +} + +TEST_F(ServeProtoTest, handshake_client_corrupted_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](const std::string toClientLog) { + for (size_t idx = 0; idx < toClientLog.size(); ++idx) { + // corrupt a copy + std::string toClientLogCorrupt = toClientLog; + toClientLogCorrupt[idx] *= 4; + ++toClientLogCorrupt[idx]; + + NullBufferedSink nullSink; + StringSource in { toClientLogCorrupt }; + + if (idx < 4 || idx == 9) { + // magic bytes don't match + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + Error); + } else if (idx < 8 || idx >= 12) { + // Number out of bounds + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + SerialisationError); + } else { + auto ver = ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"); + EXPECT_NE(ver, defaultVersion); + } + } + }); +} + } diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 91f804f0c0d..2b2e559a96a 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -280,13 +280,60 @@ VERSIONED_CHARACTERIZATION_TEST( }, .startTime = 30, .stopTime = 50, -#if 0 - // These fields are not yet serialized. - // FIXME Include in next version of protocol or document - // why they are skipped. - .cpuUser = std::chrono::milliseconds(500s), - .cpuSystem = std::chrono::milliseconds(604s), -#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_37, + "build-result-1.37", + 1 << 8 | 37, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), }, }; t; diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk index 2ee2cdb6cd7..5f7835c9f61 100644 --- a/tests/unit/libutil-support/local.mk +++ b/tests/unit/libutil-support/local.mk @@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) libutil-test-support_LIBS = libutil -libutil-test-support_LDFLAGS := -pthread -lrapidcheck +libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc index 50889cd33cc..51b9663b4c4 100644 --- a/tests/unit/libutil-support/tests/hash.cc +++ b/tests/unit/libutil-support/tests/hash.cc @@ -11,10 +11,17 @@ using namespace nix; Gen Arbitrary::arbitrary() { - Hash hash(HashAlgorithm::SHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); + Hash prototype(HashAlgorithm::SHA1); + return + gen::apply( + [](const std::vector & v) { + Hash hash(HashAlgorithm::SHA1); + assert(v.size() == hash.hashSize); + std::copy(v.begin(), v.end(), hash.hash); + return hash; + }, + gen::container>(prototype.hashSize, gen::arbitrary()) + ); } } diff --git a/tests/unit/libutil/canon-path.cc b/tests/unit/libutil/canon-path.cc index fc94ccc3d4f..bf11abe3e9c 100644 --- a/tests/unit/libutil/canon-path.cc +++ b/tests/unit/libutil/canon-path.cc @@ -80,29 +80,29 @@ namespace nix { { CanonPath p1("a//foo/bar//"); CanonPath p2("xyzzy/bla"); - ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla"); + ASSERT_EQ((p1 / p2).abs(), "/a/foo/bar/xyzzy/bla"); } { CanonPath p1("/"); CanonPath p2("/a/b"); - ASSERT_EQ((p1 + p2).abs(), "/a/b"); + ASSERT_EQ((p1 / p2).abs(), "/a/b"); } { CanonPath p1("/a/b"); CanonPath p2("/"); - ASSERT_EQ((p1 + p2).abs(), "/a/b"); + ASSERT_EQ((p1 / p2).abs(), "/a/b"); } { CanonPath p("/foo/bar"); - ASSERT_EQ((p + "x").abs(), "/foo/bar/x"); + ASSERT_EQ((p / "x").abs(), "/foo/bar/x"); } { CanonPath p("/"); - ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar"); + ASSERT_EQ((p / "foo" / "bar").abs(), "/foo/bar"); } } diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc new file mode 100644 index 00000000000..2e819ce408d --- /dev/null +++ b/tests/unit/libutil/file-content-address.cc @@ -0,0 +1,33 @@ +#include + +#include "file-content-address.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * parseFileIngestionMethod, renderFileIngestionMethod + * --------------------------------------------------------------------------*/ + +TEST(FileIngestionMethod, testRoundTripPrintParse_1) { + for (const FileIngestionMethod fim : { + FileIngestionMethod::Flat, + FileIngestionMethod::Recursive, + }) { + EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); + } +} + +TEST(FileIngestionMethod, testRoundTripPrintParse_2) { + for (const std::string_view fimS : { + "flat", + "nar", + }) { + EXPECT_EQ(renderFileIngestionMethod(parseFileIngestionMethod(fimS)), fimS); + } +} + +TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) { + EXPECT_THROW(parseFileIngestionMethod("narwhal"), UsageError); +} + +} diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index 141a55816f2..76ef86bcf72 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -66,7 +66,8 @@ TEST_F(GitTest, blob_read) { StringSource in { encoded }; StringSink out; RegularFileSink out2 { out }; - parse(out2, "", in, [](auto &, auto) {}, mockXpSettings); + ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Blob); + parseBlob(out2, "", in, false, mockXpSettings); auto expected = readFile(goldenMaster("hello-world.bin")); @@ -119,9 +120,10 @@ const static Tree tree = { TEST_F(GitTest, tree_read) { readTest("tree.bin", [&](const auto & encoded) { StringSource in { encoded }; - NullParseSink out; + NullFileSystemObjectSink out; Tree got; - parse(out, "", in, [&](auto & name, auto entry) { + ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Tree); + parseTree(out, "", in, [&](auto & name, auto entry) { auto name2 = name; if (entry.mode == Mode::Directory) name2 += '/'; @@ -193,15 +195,21 @@ TEST_F(GitTest, both_roundrip) { MemorySink sinkFiles2 { files2 }; - std::function mkSinkHook; - mkSinkHook = [&](const Path prefix, const Hash & hash) { + std::function mkSinkHook; + mkSinkHook = [&](auto prefix, auto & hash, auto executable) { StringSource in { cas[hash] }; - parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) { - mkSinkHook(prefix + "/" + name, entry.hash); - }, mockXpSettings); + parse( + sinkFiles2, prefix, in, executable, + [&](const Path & name, const auto & entry) { + mkSinkHook( + prefix + "/" + name, + entry.hash, + entry.mode == Mode::Executable); + }, + mockXpSettings); }; - mkSinkHook("", root.hash); + mkSinkHook("", root.hash, false); ASSERT_EQ(files, files2); } diff --git a/tests/unit/libutil/logging.cc b/tests/unit/libutil/logging.cc index c6dfe63d39b..1d7304f0591 100644 --- a/tests/unit/libutil/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -42,7 +42,7 @@ namespace nix { makeJSONLogger(*logger)->logEI({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -62,7 +62,7 @@ namespace nix { throw TestError(e.info()); } catch (Error &e) { ErrorInfo ei = e.info(); - ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str())); + ei.msg = HintFmt("%s; subsequent error message.", Uncolored(e.info().msg.str())); testing::internal::CaptureStderr(); logger->logEI(ei); @@ -73,7 +73,7 @@ namespace nix { } - TEST(logEI, picksUpSysErrorExitCode) { + TEST(logEI, picksUpSystemErrorExitCode) { MakeError(TestError, Error); ErrorInfo::programName = std::optional("error-unit-test"); @@ -81,12 +81,12 @@ namespace nix { try { auto x = readFile(-1); } - catch (SysError &e) { + catch (SystemError &e) { testing::internal::CaptureStderr(); logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SystemError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n"); } } @@ -176,7 +176,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foString, problem_file, 02, 13), @@ -193,7 +193,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -208,7 +208,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("hint %1%", "only"), + .msg = HintFmt("hint %1%", "only"), }); auto str = testing::internal::GetCapturedStderr(); @@ -225,7 +225,7 @@ namespace nix { logWarning({ .name = "name", - .msg = hintfmt("there was a %1%", "warning"), + .msg = HintFmt("there was a %1%", "warning"), }); auto str = testing::internal::GetCapturedStderr(); @@ -241,7 +241,7 @@ namespace nix { logWarning({ .name = "warning name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foStdin, problem_file, 2, 13), @@ -264,7 +264,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .msg = hintfmt("it has been %1% days since our last error", "zero"), + .msg = HintFmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -290,7 +290,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .msg = hintfmt("it has been %1% days since our last error", "zero"), + .msg = HintFmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -310,39 +310,39 @@ namespace nix { /* ---------------------------------------------------------------------------- - * hintfmt + * HintFmt * --------------------------------------------------------------------------*/ - TEST(hintfmt, percentStringWithoutArgs) { + TEST(HintFmt, percentStringWithoutArgs) { const char *teststr = "this is 100%s correct!"; ASSERT_STREQ( - hintfmt(teststr).str().c_str(), + HintFmt(teststr).str().c_str(), teststr); } - TEST(hintfmt, fmtToHintfmt) { + TEST(HintFmt, fmtToHintfmt) { ASSERT_STREQ( - hintfmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), + HintFmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), "the color of this this text is not yellow"); } - TEST(hintfmt, tooFewArguments) { + TEST(HintFmt, tooFewArguments) { ASSERT_STREQ( - hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(), + HintFmt("only one arg %1% %2%", "fulfilled").str().c_str(), "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " "); } - TEST(hintfmt, tooManyArguments) { + TEST(HintFmt, tooManyArguments) { ASSERT_STREQ( - hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), + HintFmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index a678dad2041..7d08f467eac 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -344,4 +344,27 @@ namespace nix { ASSERT_EQ(percentDecode(e), s); } +TEST(nix, isValidSchemeName) { + ASSERT_TRUE(isValidSchemeName("http")); + ASSERT_TRUE(isValidSchemeName("https")); + ASSERT_TRUE(isValidSchemeName("file")); + ASSERT_TRUE(isValidSchemeName("file+https")); + ASSERT_TRUE(isValidSchemeName("fi.le")); + ASSERT_TRUE(isValidSchemeName("file-ssh")); + ASSERT_TRUE(isValidSchemeName("file+")); + ASSERT_TRUE(isValidSchemeName("file.")); + ASSERT_TRUE(isValidSchemeName("file1")); + ASSERT_FALSE(isValidSchemeName("file:")); + ASSERT_FALSE(isValidSchemeName("file/")); + ASSERT_FALSE(isValidSchemeName("+file")); + ASSERT_FALSE(isValidSchemeName(".file")); + ASSERT_FALSE(isValidSchemeName("-file")); + ASSERT_FALSE(isValidSchemeName("1file")); + // regex ok? + ASSERT_FALSE(isValidSchemeName("\nhttp")); + ASSERT_FALSE(isValidSchemeName("\nhttp\n")); + ASSERT_FALSE(isValidSchemeName("http\n")); + ASSERT_FALSE(isValidSchemeName("http ")); +} + }