From 4a09c1f083679b179d3fc4c42d663ff3832469e6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2026 22:01:13 +0200 Subject: [PATCH 001/364] Don't detach the auto-GC thread It could still be running after `this` is destroyed. So we need to join it in the LocalStore destructor. --- src/libstore/gc.cc | 9 +++++++-- src/libstore/include/nix/store/local-store.hh | 1 + src/libstore/local-store.cc | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a88fe6a64acb..2d918a1391cb 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -852,12 +852,17 @@ void LocalStore::autoGC(bool sync) if (avail > state->availAfterGC * 0.97) return; + /* Note: since gcRunning is false here, any previous GC thread has exited / is exiting so the join() should be + * almost instantenous. */ + if (state->gcThread.joinable()) + state->gcThread.join(); + state->gcRunning = true; std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { + state->gcThread = std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -884,7 +889,7 @@ void LocalStore::autoGC(bool sync) // future, but we don't really care. (what??) ignoreExceptionInDestructor(); } - }).detach(); + }); } sync: diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 63a1da67d8bf..b5c80ba9e56c 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -212,6 +212,7 @@ private: */ bool gcRunning = false; std::shared_future gcFuture; + std::thread gcThread; /** * How much disk space was available after the previous diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9eb48bfe632..ef8407532bbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -432,6 +432,12 @@ LocalStore::~LocalStore() future.get(); } + { + auto state(_state->lock()); + if (state->gcThread.joinable()) + state->gcThread.join(); + } + try { auto fdTempRoots(_fdTempRoots.lock()); if (*fdTempRoots) { From f4bde1f72d3f7172366d485891b67e56eff3ed40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 8 Apr 2026 10:54:00 +0200 Subject: [PATCH 002/364] Split release artifacts into a dedicated Hydra jobset Cutting a release currently blocks on the full hydraJobs evaluation (~900 builds including sanitizers, clang-tidy, static, NixOS VM and installer tests), even though upload-release only consumes ~25 of them. On recent maintenance evals the long CI tail and darwin queue depth pushed the wait into the multi-day range while the actual artifacts were ready within an hour. Hydra hard-codes flake jobsets to outputs.hydraJobs, so the subset is exposed through a legacy jobset expression that re-enters the flake via builtins.getFlake on the locked GitHub ref. Going through the ref rather than the checked-out store path preserves rev/lastModified and thus the version suffix, keeping derivations bit-identical to the flake jobset so both share builds through the binary cache. A release aggregate job provides a single gating signal for upload-release. The release process now creates a release-$VERSION jobset alongside maintenance-$VERSION and waits on that instead of the full matrix. Requires adding https://releases.nixos.org/ to allowed-uris on hydra.nixos.org, since the locked nixpkgs input is a tarball from there and legacy jobsets run under restrict-eval. --- maintainers/release-process.md | 49 ++++++++++++---------- packaging/release-jobs.nix | 76 ++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 23 deletions(-) create mode 100644 packaging/release-jobs.nix diff --git a/maintainers/release-process.md b/maintainers/release-process.md index f8b6b6bec572..19e5e050534e 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -81,27 +81,30 @@ release: $ git push --set-upstream origin $VERSION-maintenance ``` -* Create a jobset for the release branch on Hydra as follows: - - * Go to the jobset of the previous release - (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). - - * Select `Actions -> Clone this jobset`. - - * Set identifier to `maintenance-$VERSION`. - - * Set description to `$VERSION release branch`. - - * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. - - * Hit `Create jobset`. - -* Wait for the new jobset to evaluate and build. If impatient, go to - the evaluation and select `Actions -> Bump builds to front of - queue`. - -* When the jobset evaluation has succeeded building, take note of the - evaluation ID (e.g. `1780832` in +* Create two jobsets for the release branch on Hydra: + + `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. + `release-$VERSION` builds only the artifacts consumed by + `upload-release`, so a release can be cut without waiting on the full + matrix. + + * Clone the previous `maintenance-*` jobset, set identifier + `maintenance-$VERSION`, description `$VERSION release branch`, flake + URL `github:NixOS/nix/$VERSION-maintenance`. + + * Clone the previous `release-*` jobset (or create a new **legacy** + jobset), set identifier `release-$VERSION`, description `$VERSION + release artifacts`, Nix expression `packaging/release-jobs.nix` in + input `src`, and add input `src` of type *Git checkout* pointing at + `https://github.com/NixOS/nix $VERSION-maintenance`. + +* Wait for the `release-$VERSION` jobset to evaluate and build. If + impatient, go to the evaluation and select `Actions -> Bump builds to + front of queue`. The aggregate job `release` turns green once every + required artifact is available. + +* When the release jobset evaluation has succeeded building, take note of + the evaluation ID (e.g. `1780832` in `https://hydra.nixos.org/eval/1780832`). * Tag the release: @@ -174,8 +177,8 @@ release: $ git push ``` -* Wait for the desired evaluation of the maintenance jobset to finish - building. +* Wait for the desired evaluation of the `release-$VERSION` jobset to + finish building (the `release` aggregate job is the gating signal). * Tag the release diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix new file mode 100644 index 000000000000..3da8c80de315 --- /dev/null +++ b/packaging/release-jobs.nix @@ -0,0 +1,76 @@ +# Hydra jobset containing only the artifacts consumed by +# `maintainers/upload-release.{pl,py}`, so a release can be cut without +# waiting on the full `hydraJobs` CI matrix. +# +# Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake +# jobsets to `outputs.hydraJobs`; we re-enter the flake via +# `builtins.getFlake` so derivations stay identical to the flake jobset +# and share builds through the binary cache. +# +# Hydra jobset configuration: +# Type: Legacy +# Nix expression: packaging/release-jobs.nix in input `src` +# Inputs: +# src (Git checkout) https://github.com/NixOS/nix +{ + src ? { + outPath = ./..; + }, +}: +let + # Fetch by GitHub ref rather than the bare store path Hydra hands us, + # so `rev`/`lastModified` (and thus the version suffix) match the flake + # jobset and derivations are shared. + flake = builtins.getFlake ( + if src ? rev then + "github:NixOS/nix/${src.rev}" + else + # Local evaluation / testing. + builtins.unsafeDiscardStringContext (toString src) + ); + inherit (flake) hydraJobs; + inherit (flake.inputs.nixpkgs) lib; + + jobs = { + # `nix-everything` per system: provides the store paths for + # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via + # its `doc` output. + build.nix-everything = hydraJobs.build.nix-everything; + buildCross.nix-everything.riscv64-unknown-linux-gnu = + hydraJobs.buildCross.nix-everything.riscv64-unknown-linux-gnu; + + inherit (hydraJobs) + manual + binaryTarball + binaryTarballCross + installerScript + installerScriptForGHA + dockerImage + ; + + # Aggregate gating job: green ⇒ every artifact the upload script + # needs is available. `upload-release` can wait on this single job + # instead of the whole evaluation. Constituents are referenced by + # job *name* so that an evaluation failure in one of them does not + # take down the aggregate's own evaluation. + release = flake.inputs.nixpkgs.legacyPackages.x86_64-linux.releaseTools.aggregate { + name = "nix-release-${flake.packages.x86_64-linux.nix-everything.version}"; + meta.description = "Artifacts required for a Nix release"; + constituents = + let + collectJobNames = + prefix: x: + if lib.isDerivation x then + [ prefix ] + else if lib.isAttrs x then + lib.concatLists ( + lib.mapAttrsToList (n: collectJobNames (if prefix == "" then n else "${prefix}.${n}")) x + ) + else + [ ]; + in + collectJobNames "" (builtins.removeAttrs jobs [ "release" ]); + }; + }; +in +jobs From 892d870d9640fe5a51b0e5e38107b27ab276c2f7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Apr 2026 22:11:52 +0300 Subject: [PATCH 003/364] libstore: Plug thread safety issues in recursive-nix --- .../include/nix/store/restricted-store.hh | 52 +++++++++++++++---- src/libstore/restricted-store.cc | 15 ++++-- .../unix/build/chroot-derivation-builder.cc | 2 - src/libstore/unix/build/derivation-builder.cc | 19 ++++--- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index ca4e0b8536a7..79167a3d08c8 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -2,6 +2,9 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/sync.hh" + +#include namespace nix { @@ -28,15 +31,20 @@ struct RestrictionContext */ virtual const StorePathSet & originalPaths() = 0; - /** - * Paths that were added via recursive Nix calls. - */ - StorePathSet addedPaths; + struct State + { + /** + * Paths that were added via recursive Nix calls. + */ + std::map> addedPaths; - /** - * Realisations that were added via recursive Nix calls. - */ - std::set addedDrvOutputs; + /** + * Realisations that were added via recursive Nix calls. + */ + std::set addedDrvOutputs; + }; + + Sync state_; /** * Recursive Nix calls are only allowed to build or realize paths @@ -56,7 +64,33 @@ struct RestrictionContext { if (isAllowed(path)) return; - addDependencyImpl(path); + + std::promise promise; + + auto [future, shouldAdd] = [&]() -> std::pair, bool> { + auto state(state_.lock()); + if (auto iter = state->addedPaths.find(path); iter != state->addedPaths.end()) { + return {iter->second, false}; + } + auto [iter2, _] = state->addedPaths.emplace(path, promise.get_future().share()); + return {iter2->second, true}; + }(); + + /* Another daemon worker thread already started adding the dependency. Just wait for it + to complete. */ + if (!shouldAdd) { + future.get(); + return; + } + + try { + addDependencyImpl(path); + promise.set_value(); + } catch (...) { + /* Notify all other waiters that we are done. */ + promise.set_exception(std::current_exception()); + throw; + } } virtual ~RestrictionContext() = default; diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 8001d43ec8d8..03c130da9653 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -167,8 +167,11 @@ StorePathSet RestrictedStore::queryAllValidPaths() StorePathSet paths; for (auto & p : goal.originalPaths()) paths.insert(p); - for (auto & p : goal.addedPaths) - paths.insert(p); + for (auto & [p, future] : goal.state_.lock()->addedPaths) { + /* Only report the paths that have finished materialising in the sandbox. */ + if (future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + paths.insert(p); + } return paths; } @@ -300,8 +303,12 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : newRealisations) - goal.addedDrvOutputs.insert(real.id); + + { + auto state(goal.state_.lock()); + for (auto & real : newRealisations) + state->addedDrvOutputs.insert(real.id); + } return results; } diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index d10d98245a11..157d9e319173 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -131,8 +131,6 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl std::pair addDependencyPrep(const StorePath & path) { - DerivationBuilderImpl::addDependencyImpl(path); - debug("materialising '%s' in the sandbox", store.printStorePath(path)); std::filesystem::path source = store.toRealPath(path); diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 85aa98c7ae87..3cf7977c50d2 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -238,12 +238,18 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder bool isAllowed(const StorePath & path) override { - return inputPaths.count(path) || addedPaths.count(path); + if (inputPaths.count(path)) + return true; + auto state(state_.lock()); + auto iter = state->addedPaths.find(path); + if (iter == state->addedPaths.end()) + return false; + return iter->second.wait_for(std::chrono::seconds(0)) == std::future_status::ready; } bool isAllowed(const DrvOutput & id) override { - return addedDrvOutputs.count(id); + return state_.lock()->addedDrvOutputs.count(id); } bool isAllowed(const DerivedPath & req); @@ -1165,7 +1171,7 @@ void DerivationBuilderImpl::startDaemon() ref(std::dynamic_pointer_cast(this->store.shared_from_this())), *this); - addedPaths.clear(); + state_.lock()->addedPaths.clear(); auto socketName = ".nix-socket"; std::filesystem::path socketPath = tmpDir / socketName; @@ -1246,10 +1252,7 @@ void DerivationBuilderImpl::stopDaemon() daemonSocket.close(); } -void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) -{ - addedPaths.insert(path); -} +void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) {} void DerivationBuilderImpl::chownToBuilder(const std::filesystem::path & path) { @@ -1434,7 +1437,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() referenceablePaths.insert(p); for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) + for (auto & [p, _] : state_.lock()->addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and make all From 099fd303c0b94e717fe70c0eedfc3ea37b3d038a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:20 +0300 Subject: [PATCH 004/364] libstore: Reduce memory usage of the Worker even more Several things. Derivation is the primary memory hog in the build loop, with the total memory usage getting into gigabytes just to hold the derivations in memory. Previously I've optimised most goals to share a single ref, but turns out I missed DerivationResolutionGoal. Also fixes a bug in the move assignment of Goal::Co that didn't destroy the coroutine frame of the Goal being moved into. Just this single commit lowers the memory usage of the build loop by 25% in my tests (from 4.2G -> 3.2G) on a 70k derivation closure. --- src/libstore/build/derivation-goal.cc | 17 +++++++---------- .../build/derivation-resolution-goal.cc | 4 ++-- src/libstore/build/goal.cc | 9 +++++++-- src/libstore/build/worker.cc | 2 +- .../store/build/derivation-resolution-goal.hh | 5 +++-- src/libstore/include/nix/store/build/goal.hh | 4 +++- src/libstore/include/nix/store/build/worker.hh | 2 +- 7 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6e2d3223b10f..6d76cd9d275e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -144,11 +144,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) worker.store.printStorePath(drvPath)); } - auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode); - { - Goals waitees{resolutionGoal}; - co_await await(std::move(waitees)); - } + auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, drv, buildMode); + co_await await({resolutionGoal}); + if (nrFailed != 0) { co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); } @@ -219,6 +217,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(false); } + /* We don't need it any more and don't want to hold on to it while suspended. */ + resolutionGoal.reset(); + /* Give up on substitution for the output we want, actually build this derivation */ auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); @@ -226,11 +227,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; - { - Goals waitees; - waitees.insert(g); - co_await await(std::move(waitees)); - } + co_await await({g}); trace("outer build done"); diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 81c698e18563..1343267eff31 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -7,10 +7,10 @@ namespace nix { DerivationResolutionGoal::DerivationResolutionGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) : Goal(worker, resolveDerivation()) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv(std::move(drv)) , buildMode{buildMode} { name = fmt("resolving derivation '%s'", worker.store.printStorePath(drvPath)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 946314afe140..1bfa563d159a 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -65,10 +65,15 @@ Co::Co(Co && rhs) rhs.handle = nullptr; } -void Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) { - this->handle = rhs.handle; + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } + handle = rhs.handle; rhs.handle = nullptr; + return *this; } Co::~Co() diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0c63beb4dd1e..002bb1a30d79 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -104,7 +104,7 @@ std::shared_ptr Worker::makeDerivationGoal( } std::shared_ptr -Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode) +Worker::makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode); } diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index 843e4031aa7e..972558e9706f 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -37,7 +37,8 @@ struct DerivationResolutionGoal : public Goal { friend class Worker; - DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); + DerivationResolutionGoal( + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); /** * If the derivation needed to be resolved, this is resulting @@ -55,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 0b7367ff7e02..f8293609e11c 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -233,8 +233,10 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - void operator=(Co &&); + Co & operator=(Co &&); Co(Co && rhs); + Co & operator=(const Co &) = delete; + Co(const Co & rhs) = delete; ~Co(); bool await_ready() diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 4c836986811f..12e6c0122e1c 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -265,7 +265,7 @@ public: * @ref DerivationResolutionGoal "derivation resolution goal" */ std::shared_ptr - makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode); + makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref DerivationBuildingGoal "derivation building goal" From 88c137708fc581dd64c67a9ae6fa3b452af6eb3f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:32 +0300 Subject: [PATCH 005/364] libstore: Move childEvents from the promise into Goal itself ChildEvents has no business being in the coroutines promise type - it bloats it unnecessarily. The whole promise type was like 500 bytes just because it stored TimedOut inline - now it's behind a unique_ptr as it should have been. Shaves off like 300MB of memory usage on my stress-testing benchmark. --- .../build/derivation-building-goal.cc | 8 +- src/libstore/build/goal.cc | 24 +++--- src/libstore/include/nix/store/build/goal.hh | 82 +++++++++---------- 3 files changed, 55 insertions(+), 59 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 57a4017e8e24..248ef7507c4e 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -764,9 +764,9 @@ Goal::Co DerivationBuildingGoal::buildWithHook( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -1035,9 +1035,9 @@ Goal::Co DerivationBuildingGoal::buildLocally( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 1bfa563d159a..7651282e3e4f 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -12,16 +12,15 @@ TimedOut::TimedOut(time_t maxDuration) using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; -using ChildEvents = decltype(promise_type::childEvents); -void ChildEvents::pushChildEvent(ChildOutput event) +void Goal::ChildEvents::pushChildEvent(ChildOutput event) { if (childTimeout) return; // Already timed out, ignore childOutputs.push(std::move(event)); } -void ChildEvents::pushChildEvent(ChildEOF event) +void Goal::ChildEvents::pushChildEvent(ChildEOF event) { if (childTimeout) return; // Already timed out, ignore @@ -29,20 +28,20 @@ void ChildEvents::pushChildEvent(ChildEOF event) childEOF = std::move(event); } -void ChildEvents::pushChildEvent(TimedOut event) +void Goal::ChildEvents::pushChildEvent(TimedOut event) { // Timeout is immediate - flush pending events childOutputs = {}; childEOF.reset(); - childTimeout = std::move(event); + childTimeout = std::make_unique(std::move(event)); } -bool ChildEvents::hasChildEvent() const +bool Goal::ChildEvents::hasChildEvent() const { return !childOutputs.empty() || childEOF || childTimeout; } -Goal::ChildEvent ChildEvents::popChildEvent() +Goal::ChildEvent Goal::ChildEvents::popChildEvent() { if (!childOutputs.empty()) { auto event = std::move(childOutputs.front()); @@ -52,7 +51,7 @@ Goal::ChildEvent ChildEvents::popChildEvent() if (childEOF) return *std::exchange(childEOF, std::nullopt); if (childTimeout) - return *std::exchange(childTimeout, std::nullopt); + return std::exchange(childTimeout, nullptr); unreachable(); } @@ -278,22 +277,19 @@ void Goal::work() void Goal::handleChildOutput(Descriptor fd, std::string_view data) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); worker.wakeUp(shared_from_this()); } void Goal::handleEOF(Descriptor fd) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + childEvents.pushChildEvent(ChildEOF{fd}); worker.wakeUp(shared_from_this()); } void Goal::timedOut(TimedOut && ex) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + childEvents.pushChildEvent(std::move(ex)); worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index f8293609e11c..6ddc73250d34 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -74,7 +74,43 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant>; + private: + class ChildEvents + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::unique_ptr childTimeout; + + public: + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + }; + /** * Goals that this goal is waiting for. */ @@ -85,6 +121,8 @@ private: */ std::optional cachedKey; + ChildEvents childEvents; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -152,22 +190,6 @@ public: friend Goal; }; - /** - * Event types for child process communication, delivered via coroutines. - */ - struct ChildOutput - { - Descriptor fd; - std::string data; - }; - - struct ChildEOF - { - Descriptor fd; - }; - - using ChildEvent = std::variant; - /** * Tag type for `co_await`-ing child events. * Returns a `ChildEvent` when resumed. @@ -321,28 +343,6 @@ public: */ bool alive = true; - class - { - /** - * Structured queue of child events: - * - outputs: stream of data from child - * - eof: optional end-of-stream marker - * - timeout: optional timeout that flushes/overrides other events - */ - std::queue childOutputs; - std::optional childEOF; - std::optional childTimeout; - - public: - - void pushChildEvent(ChildOutput event); - void pushChildEvent(ChildEOF event); - void pushChildEvent(TimedOut event); - bool hasChildEvent() const; - ChildEvent popChildEvent(); - - } childEvents; - /** * The awaiter used by @ref final_suspend. */ @@ -450,7 +450,7 @@ public: bool await_ready() { - assert(!promise.childEvents.hasChildEvent()); + assert(!promise.goal->childEvents.hasChildEvent()); return false; } @@ -478,7 +478,7 @@ public: bool await_ready() { - return handle && handle.promise().childEvents.hasChildEvent(); + return handle && handle.promise().goal->childEvents.hasChildEvent(); } void await_suspend(handle_type h) @@ -489,7 +489,7 @@ public: ChildEvent await_resume() { assert(handle); - return handle.promise().childEvents.popChildEvent(); + return handle.promise().goal->childEvents.popChildEvent(); } }; From 1cded5e512927a79a6e60b5a68c97aca4c1ef0a4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:39 +0300 Subject: [PATCH 006/364] derivation-resolution-goal: Make use of deducing this for a recursive lambda, get rid of allocationsA std::function allocates for the captures, but with C++23 we have a much better way of writing recursive lambdas - so use it. --- .../build/derivation-resolution-goal.cc | 66 +++++++++---------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 1343267eff31..f457c5614a6f 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -40,42 +40,38 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() std::map, GoalPtr, value_comparison> inputGoals; - { - std::function, const DerivedPathMap::ChildNode &)> - addWaiteeDerivedPath; - - addWaiteeDerivedPath = [&](ref inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto g = worker.makeGoal( - DerivedPath::Built{ - .drvPath = inputDrv, - .outputs = inputNode.value, - }, - buildMode == bmRepair ? bmRepair : bmNormal); - inputGoals.insert_or_assign(inputDrv, g); - waitees.insert(std::move(g)); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - addWaiteeDerivedPath( - make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); - }; - - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { - /* Ensure that pure, non-fixed-output derivations don't - depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() - && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (inputDrv.type().isImpure()) - throw Error( - "pure derivation '%s' depends on impure derivation '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(inputDrvPath)); - } - - addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); + auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( + this const auto & self, + ref inputDrv, + const DerivedPathMap::ChildNode & inputNode) -> void { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( + DerivedPath::Built{ + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); } + for (const auto & [outputName, childNode] : inputNode.childMap) + self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() + && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) + throw Error( + "pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(inputDrvPath)); + } + + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } co_await await(std::move(waitees)); From 50923476b9dca028051263922e607a1a1887abd8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:47 +0300 Subject: [PATCH 007/364] derivation-resolution-goal: Replace value_comparison with a lambda Having such helpers in the namespace scope in a translation unit is a bad time when using unity builds. At some point this was duplicated between several files and caused name collisions with unity builds. A lambda is much cleaner and doesn't have the same footgun. Lambdas can have template parameters too now. --- .../build/derivation-resolution-goal.cc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index f457c5614a6f..6f55ea29d96e 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -22,23 +22,16 @@ std::string DerivationResolutionGoal::key() return "dc$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -/** - * Used for `inputGoals` local variable below - */ -struct value_comparison -{ - template - bool operator()(const ref & lhs, const ref & rhs) const - { - return *lhs < *rhs; - } -}; - Goal::Co DerivationResolutionGoal::resolveDerivation() { Goals waitees; - std::map, GoalPtr, value_comparison> inputGoals; + using ValueComparison = decltype([](const ref & lhs, const ref & rhs) { + /* Compare the values, not the pointers themselves. */ + return *lhs < *rhs; + }); + + std::map, GoalPtr, ValueComparison> inputGoals; auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( this const auto & self, From 136e1a2c93e153eb05518f4175aa453071a443b6 Mon Sep 17 00:00:00 2001 From: dramforever Date: Tue, 7 Apr 2026 15:53:57 +0800 Subject: [PATCH 008/364] fetchClosure: Add temproot before checking path This avoids the fetched path from getting garbage collected if it is already valid. --- src/libexpr/primops/fetchClosure.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index db182ab499b2..fe3873c101b4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -23,6 +23,8 @@ static void runFetchClosureWithRewrite( const std::optional & toPathMaybe, Value & v) { + if (toPathMaybe) + state.store->addTempRoot(*toPathMaybe); // establish toPath or throw @@ -74,6 +76,7 @@ static void runFetchClosureWithRewrite( static void runFetchClosureWithContentAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); @@ -103,6 +106,7 @@ static void runFetchClosureWithContentAddressedPath( static void runFetchClosureWithInputAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); From 7fe57da4b294f8e28f6155194293dde1f59a33ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 20 Apr 2026 13:43:08 +0000 Subject: [PATCH 009/364] libstore: don't print URL userinfo in FileTransfer diagnostics When a URL with embedded credentials (scheme://user:pass@host/...) is passed to builtins.fetchurl or any other downloadFile caller, the password is printed verbatim in progress, retry-warning and error messages because they all format request.uri directly: warning: unable to download 'http://user:SECRET@host/file': ...; retrying in 256 ms Add FileTransferRequest::displayUri() which returns uri with the userinfo component stripped, and use it at every diagnostic call site. request.uri itself is unchanged so CURLOPT_URL, result.urls and fetcher-cache keys are unaffected. --- src/libstore-tests/filetransfer-request.cc | 19 +++++++++ src/libstore-tests/meson.build | 1 + src/libstore/filetransfer.cc | 42 +++++++++++++------ .../include/nix/store/filetransfer.hh | 8 ++++ tests/nixos/fetchurl.nix | 20 +++++++++ 5 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 src/libstore-tests/filetransfer-request.cc diff --git a/src/libstore-tests/filetransfer-request.cc b/src/libstore-tests/filetransfer-request.cc new file mode 100644 index 000000000000..b89bb7ed0bc1 --- /dev/null +++ b/src/libstore-tests/filetransfer-request.cc @@ -0,0 +1,19 @@ +#include + +#include "nix/store/filetransfer.hh" + +namespace nix { + +TEST(FileTransferRequest, displayUriStripsUserinfo) +{ + FileTransferRequest req(VerbatimURL{std::string{"https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"}}); + // uri itself is untouched (used for CURLOPT_URL, result.urls, cache keys). + EXPECT_EQ(req.uri.to_string(), "https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"); + // displayUri() drops the userinfo for diagnostics. + EXPECT_EQ(req.displayUri(), "https://example.org:8443/path/file.toml?x=1"); + + FileTransferRequest plain(VerbatimURL{std::string{"https://example.org/file"}}); + EXPECT_EQ(plain.displayUri(), "https://example.org/file"); +} + +} // namespace nix diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index a126a87aca8b..78a1cc4a12dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -64,6 +64,7 @@ sources = files( 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-request.cc', 'filetransfer-retry.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 85bc650dc8d4..6d97fb4e3f5d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -281,7 +281,11 @@ struct curlFileTransfer : public FileTransfer try { if (!done && enqueued) fail(FileTransferError( - Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri)); + Interrupted, + {}, + "%s of '%s' was interrupted", + Uncolored(request.noun()), + request.displayUri())); } catch (...) { ignoreExceptionInDestructor(); } @@ -297,7 +301,7 @@ struct curlFileTransfer : public FileTransfer /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ - e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); + e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.displayUri()); } catch (...) { /* Can't add more context to the error. */ } @@ -361,7 +365,7 @@ struct curlFileTransfer : public FileTransfer try { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.displayUri(), trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); if (std::smatch match; std::regex_match(line, match, statusLine)) { @@ -450,8 +454,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), - Logger::Fields{request.uri.to_string()}, + fmt("%s '%s'", request.verb(/*continuous=*/true), request.displayUri()), + Logger::Fields{request.displayUri()}, request.parentAct); // Reset the start time to when we actually started the download. startTime = std::chrono::steady_clock::now(); @@ -716,7 +720,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", Uncolored(request.noun()), - request.uri, + request.displayUri(), code, httpStatus, result.bodySize, @@ -815,14 +819,14 @@ struct curlFileTransfer : public FileTransfer std::move(response), "%s of '%s' was interrupted", Uncolored(request.noun()), - request.uri) + request.displayUri()) : httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", Uncolored(request.verb()), - request.uri, + request.displayUri(), httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError( @@ -830,7 +834,7 @@ struct curlFileTransfer : public FileTransfer std::move(response), "unable to %s '%s': %s (%d) %s", Uncolored(request.verb()), - request.uri, + request.displayUri(), curl_easy_strerror(code), code, errbuf); @@ -1081,7 +1085,7 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.displayUri()); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; @@ -1133,7 +1137,7 @@ struct curlFileTransfer : public FileTransfer { if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" && item->request.uri.scheme() != "s3") - throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); + throw nix::Error("uploading to '%s' is not supported", item->request.displayUri()); { auto state(state_.lock()); @@ -1194,6 +1198,20 @@ ref makeFileTransfer(const FileTransferSettings & settings) return makeCurlFileTransfer(settings); } +std::string FileTransferRequest::displayUri() const +{ + try { + auto parsed = uri.parsed(); + if (parsed.authority && parsed.authority->user) { + parsed.authority->user.reset(); + parsed.authority->password.reset(); + return parsed.to_string(); + } + } catch (BadURL &) { + } + return uri.to_string(); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1283,7 +1301,7 @@ void FileTransfer::download( state->request.notify_one(); }); - request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { + request.dataCallback = [_state, uri = request.displayUri()](std::string_view data) -> PauseTransfer { auto state(_state->lock()); if (state->quit) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index a423249e6634..2bd9ce201daa 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -307,6 +307,14 @@ struct FileTransferRequest { } + /** + * `uri` with any userinfo (`user:password@`) stripped, for use in + * progress, warning and error messages so credentials embedded in + * the URL don't leak into logs. Returns `uri` verbatim if it can't + * be parsed. + */ + std::string displayUri() const; + /** * Returns the method description for logging purposes. */ diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index e8663debbcd4..b3e744424e3c 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -54,6 +54,14 @@ in echo 'foobar' > "$out/index.html" ''; }; + + virtualHosts."auth" = { + basicAuth.alice = "s3cr3t"; + root = pkgs.runCommand "nginx-root" { } '' + mkdir "$out" + echo 'authed' > "$out/index.html" + ''; + }; }; security.pki.certificateFiles = [ "${goodCert}/cert.pem" ]; @@ -61,6 +69,7 @@ in networking.hosts."127.0.0.1" = [ "good" "bad" + "auth" ]; virtualisation.writableStore = true; @@ -89,5 +98,16 @@ in # Fetching from a server with a trusted cert should work via environment variable override. machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'") + + # builtins.fetchurl should authenticate using userinfo from the URL. + out = machine.succeed("nix eval --raw --impure --no-substitute --expr 'builtins.readFile (builtins.fetchurl { url = \"http://alice:s3cr3t@auth/index.html\"; sha256 = \"sha256-67y3HalfTt2zfgMt7BwU5vkaGWcelFlR4jryIkA1dTo=\"; })'") + assert out == "authed\n", out + + # On failure the transport-layer diagnostic must show the stripped URL, + # not the userinfo. (The eval trace still echoes the source expression; + # that is the user's own input, not a leak.) + err = machine.fail("nix eval --raw --impure --no-substitute --option tarball-ttl 0 --expr 'builtins.fetchurl { url = \"http://alice:wrong-secret@auth/index.html\"; sha256 = \"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"; }' 2>&1") + print(err) + assert "unable to download 'http://auth/index.html'" in err, err ''; } From f0d9109fa355d59ffdd5f07220e5b3d396256360 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:10:54 +0300 Subject: [PATCH 010/364] libexpr: Fix error message in InvalidPathError It was only printing the base name, which isn't how we usually print store paths. --- src/libexpr/eval-error.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 45cb1e409dd1..67e8d5ea1ded 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,11 +1,12 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" +#include "nix/store/store-api.hh" namespace nix { InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) - : CloneableError(state, "path '%s' is not valid", path.to_string()) + : CloneableError(state, "path '%s' is not valid", state.store->printStorePath(path)) , path{path} { } From 2949729038395be70f99173388e992244e4678f0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 21:38:23 +0300 Subject: [PATCH 011/364] libflake: Fix argument order evaluation footgun mountInput modifies lockedRef.input to stuff narHash into it. --- src/libflake/flake.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index deb1c16b71e9..45871cff87ee 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -391,14 +391,9 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } + auto rootDir = state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)); // Re-parse flake.nix from the store. - return readFlake( - state, - originalRef, - resolvedRef, - lockedRef, - state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)), - lockRootAttrPath); + return readFlake(state, originalRef, resolvedRef, lockedRef, rootDir, lockRootAttrPath); } Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) From 49b2680eaf80bda3759139df6094dd979bc2b1b0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:52:59 +0300 Subject: [PATCH 012/364] nix flake archive: Use 'deducing this' recursive lambda instead of std::function One slight blemish I noticed while touching this code. With C++23 we can simplify things. --- src/nix/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 42de78453451..15e9700d9d18 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1095,8 +1095,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs sources.insert(storePath); // FIXME: use graph output, handle cycles. - std::function traverse; - traverse = [&](const flake::Node & node) { + auto traverse = [&store, json = json, dryRun = dryRun, &sources]( + this const auto & self, const flake::Node & node) -> nlohmann::json { nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { @@ -1110,9 +1110,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs auto & jsonObj3 = jsonObj2[inputName]; if (storePath) jsonObj3["path"] = store->printStorePath(*storePath); - jsonObj3["inputs"] = traverse(**inputNode); + jsonObj3["inputs"] = self(**inputNode); } else - traverse(**inputNode); + self(**inputNode); } } return jsonObj2; From d5f162d1166f71a8eb1e343727a51fcfb8a88235 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 15:12:45 +0300 Subject: [PATCH 013/364] libexpr-c: Fix UAF on readOnlyMode Turns out the readOnlyMode was always dangling in the C API and nobody noticed... Previous commits just started accessing it, which was caught by ASan. This is the minimal fix I can think of. --- src/libexpr-c/nix_api_expr.cc | 21 ++++++++----------- src/libexpr-c/nix_api_expr_internal.h | 3 +-- src/libexpr/eval-settings.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/include/nix/expr/eval-settings.hh | 9 +++++++- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 97680ac6bfe7..99c8e42cc2ba 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -129,23 +129,20 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - }); + auto readOnly = nix::make_ref(true); + return new nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ *readOnly}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = readOnly, + }; } NIXC_CATCH_ERRS_NULL } void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - if (builder) - builder->~nix_eval_state_builder(); - operator delete(builder, static_cast(alignof(nix_eval_state_builder))); + delete builder; } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -154,7 +151,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil context->last_err_code = NIX_OK; try { // TODO: load in one go? - builder->settings.readOnlyMode = nix::settings.readOnlyMode; + builder->settings.readOnlyMode = &nix::settings.readOnlyMode; loadConfFile(builder->settings); loadConfFile(builder->fetchSettings); } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b38aeaf7b498..3f7e4bf1df12 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -18,8 +18,7 @@ struct nix_eval_state_builder nix::EvalSettings settings; nix::fetchers::Settings fetchSettings; nix::LookupPath lookupPath; - // TODO: make an EvalSettings setting own this instead? - bool readOnlyMode; + nix::ref readOnlyMode; }; struct EvalState diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 5cf0ae04304e..a6a3e829f20c 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -70,7 +70,7 @@ Strings EvalSettings::parseNixPath(const std::string & s) } EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) - : readOnlyMode{readOnlyMode} + : readOnlyMode{&readOnlyMode} , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_ABORT_ON_WARN"); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d66b8ead1c9..e4fe3ba6a5f8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2583,7 +2583,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), ContentAddressMethod::Raw::NixArchive, nullptr, diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index d9dba95370b1..a03a8dadbef1 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -71,7 +71,14 @@ struct EvalSettings : Config EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); - bool & readOnlyMode; + /* FIXME: This really shouldn't be public. The C API should have non-global settings instead. */ + bool * readOnlyMode = nullptr; + + bool isReadOnly() const + { + assert(readOnlyMode); + return *readOnlyMode; + } static Strings getDefaultNixPath(); From d34b442634de5d27a221e4c5885724f271228fcd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Apr 2026 21:40:25 +0300 Subject: [PATCH 014/364] flake archive: Factor out storePath computation/fetching into a lambda This is necessary for making flake store paths lazier and also slightly more concise anyway. --- src/nix/flake.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 15e9700d9d18..2a5239549e0f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1090,20 +1090,25 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto getStorePath = [&](const FlakeRef & lockedRef) { + return dryRun ? lockedRef.input.computeStorePath(*store) + : std::get(lockedRef.input.fetchToStore(fetchSettings, *store)); + }; + + auto storePath = getStorePath(flake.flake.lockedRef); sources.insert(storePath); // FIXME: use graph output, handle cycles. - auto traverse = [&store, json = json, dryRun = dryRun, &sources]( + auto traverse = [&store, json = json, &sources, &getStorePath]( this const auto & self, const flake::Node & node) -> nlohmann::json { nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { std::optional storePath; - if (!(*inputNode)->lockedRef.input.isRelative()) { - storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store).first; + const auto & lockedRef = (*inputNode)->lockedRef; + if (!lockedRef.input.isRelative()) { + storePath = getStorePath(lockedRef); sources.insert(*storePath); } if (json) { From 9fcb58f388611e0660db51fed4195594eb21a985 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Apr 2026 21:59:22 +0300 Subject: [PATCH 015/364] libexpr-c: Remove unsafe_new_with_self This is thankfully not used by anything else anymore. Good riddance, since all usages of it had bugs in them. --- src/libexpr-c/nix_api_expr.cc | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 99c8e42cc2ba..2387ee8c38c2 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,27 +19,6 @@ # include #endif -/** - * @brief Allocate and initialize using self-reference - * - * This allows a brace initializer to reference the object being constructed. - * - * @warning Use with care, as the pointer points to an object that is not fully constructed yet. - * - * @tparam T Type to allocate - * @tparam F A function type for `init`, taking a T* and returning the initializer for T - * @param init Function that takes a T* and returns the initializer for T - * @return Pointer to allocated and initialized object - */ -template -static T * unsafe_new_with_self(F && init) -{ - // Allocate - void * p = ::operator new(sizeof(T), static_cast(alignof(T))); - // Initialize with placement new - return new (p) T(init(static_cast(p))); -} - extern "C" { nix_err nix_libexpr_init(nix_c_context * context) From 7d12269604077f6f302f75449c20c50d38b03f7f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:11:38 +0300 Subject: [PATCH 016/364] libfetchers: Use source accessor if the mercurial fetcher, use makeFSSourceAccessor This significantly simplifies path filtering and gets rid of raw file system accesses. --- src/libfetchers/mercurial.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index af5c94c1b494..ab1d31ed330f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -240,26 +240,25 @@ struct MercurialInputScheme : InputScheme }), "\0"s); - auto actualPath = absPath(localPath); + /* FIXME: Check that the access to this path is allowed. */ + auto accessor = makeFSSourceAccessor(absPath(localPath)); PathFilter filter = [&](const std::string & p) -> bool { - assert(hasPrefix(p, actualPath.string())); - std::string file(p, actualPath.string().size() + 1); + auto cp = CanonPath(p); + auto st = accessor->lstat(cp); - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; + if (st.type == SourceAccessor::tDirectory) { + auto prefix = cp.rel() + "/"; auto i = files.lower_bound(prefix); return i != files.end() && hasPrefix(*i, prefix); } - return files.count(file); + return files.count(cp.rel()); }; return store.addToStore( input.getName(), - {getFSSourceAccessor(), CanonPath(actualPath.string())}, + {accessor, CanonPath::root}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, @@ -381,7 +380,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir / ".hg_archival.txt"); - auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root}); Attrs infoAttrs({ {"revCount", (uint64_t) revCount}, From 48100ab18c99a63c195d1d17f249fe9fc685b60f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 00:14:41 +0300 Subject: [PATCH 017/364] libutil: Add a dirFd callback for iterative openFileEnsureBeneathNoSymlinks This would be useful for caching parent directory file descriptors in the source accessor. I don't think the windows code works too well now, so I left the callback there as a FIXME to be called. It's just a perf improvement in general and shouldn't break if it never gets called. --- .../include/nix/util/file-system-at.hh | 8 ++++--- src/libutil/unix/file-system-at.cc | 23 +++++++++++++++---- src/libutil/windows/file-system-at.cc | 8 ++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 0f4678461ca0..17cd8b46688a 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -90,6 +90,8 @@ OsString readLinkAt(Descriptor dirFd, const CanonPath & path); * @param flags (Unix) `O_*` flags (must not include `O_NOFOLLOW`) * @param mode (Unix) Mode for `O_{CREAT,TMPFILE}` * + * @param dirFdCallback Callback invoked that gets the ownership of an intermediate directory fd. + * * @pre `path.isRoot()` is false * * @throws SymlinkNotAllowed if an interior path component is a @@ -118,12 +120,12 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( #ifdef _WIN32 ACCESS_MASK desiredAccess, ULONG createOptions, - ULONG createDisposition = FILE_OPEN + ULONG createDisposition = FILE_OPEN, #else int flags, - mode_t mode = 0 + mode_t mode = 0, #endif -); + std::function dirFdCallback = nullptr); #ifdef __linux__ namespace linux { diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 762a245ded04..11c7bcd064fa 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -140,20 +140,27 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t } } -static AutoCloseFD -openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +static AutoCloseFD openFileEnsureBeneathNoSymlinksIterative( + Descriptor dirFd, + const CanonPath & path, + int flags, + mode_t mode, + std::function dirFdCallback) { AutoCloseFD parentFd; auto nrComponents = std::ranges::distance(path); assert(nrComponents >= 1); auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */ auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; }; + auto currentRelPath = CanonPath::root; /* This rather convoluted loop is necessary to avoid TOCTOU when validating that no inner path component is a symlink. */ for (auto it = components.begin(); it != components.end(); ++it) { auto component = std::string(*it); /* Copy into a string to make NUL terminated. */ assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */ + auto prevRelPath = currentRelPath; + currentRelPath = currentRelPath / *it; AutoCloseFD parentFd2 = ::openat( getParentFd(), /* First iteration uses dirFd. */ @@ -188,6 +195,9 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat return AutoCloseFD{}; } + if (dirFdCallback && parentFd) + dirFdCallback(std::move(parentFd), std::move(prevRelPath)); + parentFd = std::move(parentFd2); } @@ -221,7 +231,12 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat return res; } -AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +AutoCloseFD openFileEnsureBeneathNoSymlinks( + Descriptor dirFd, + const CanonPath & path, + int flags, + mode_t mode, + std::function dirFdCallback) { /* Just in case the invariant is somehow broken. */ assert(!path.rel().starts_with('/')); @@ -263,7 +278,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & } #endif - return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); + return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode, std::move(dirFdCallback)); } OsString readLinkAt(Descriptor dirFd, const CanonPath & path) diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index 668bd7eb402f..54248f2a36ca 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -229,7 +229,13 @@ PosixStat fstat(Descriptor fd) } AutoCloseFD openFileEnsureBeneathNoSymlinks( - Descriptor dirFd, const CanonPath & path, ACCESS_MASK desiredAccess, ULONG createOptions, ULONG createDisposition) + Descriptor dirFd, + const CanonPath & path, + ACCESS_MASK desiredAccess, + ULONG createOptions, + ULONG createDisposition, + /* FIXME: Actually call this callback. */ + [[maybe_unused]] std::function dirFdCallback) { assert(!path.isRoot()); assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ From 293aa8ded34dd79d04217505897b678a9d7bc522 Mon Sep 17 00:00:00 2001 From: Krish Jaiswal Date: Tue, 21 Apr 2026 02:52:45 +0530 Subject: [PATCH 018/364] Add redirect for language/values.html --- doc/manual/source/_redirects | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 07b3130f9ce9..7e4557f7d595 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -36,6 +36,7 @@ /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! /language/values /language/types 301! +/language/values.html /language/types 301! /language/constructs /language/syntax 301! /language/builtin-constants /language/builtins 301! From e069dae4ef9f7e97cf5869f9c0c124979b54364a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 00:01:32 +0200 Subject: [PATCH 019/364] release-jobs: include all buildCross.nix-everything targets Avoids drifting from upload-release.pl when new cross targets such as x86_64-unknown-freebsd are added to fallback-paths. Also drop the reference to a not-yet-existing Python rewrite of the upload script. Addresses review comments on #15640. --- packaging/release-jobs.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index 3da8c80de315..23add3905e33 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -1,5 +1,5 @@ # Hydra jobset containing only the artifacts consumed by -# `maintainers/upload-release.{pl,py}`, so a release can be cut without +# `maintainers/upload-release.pl`, so a release can be cut without # waiting on the full `hydraJobs` CI matrix. # # Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake @@ -36,8 +36,7 @@ let # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via # its `doc` output. build.nix-everything = hydraJobs.build.nix-everything; - buildCross.nix-everything.riscv64-unknown-linux-gnu = - hydraJobs.buildCross.nix-everything.riscv64-unknown-linux-gnu; + buildCross.nix-everything = hydraJobs.buildCross.nix-everything; inherit (hydraJobs) manual From 7981f28017b7af317546749e25bddb412b3f4c52 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 01:54:04 +0300 Subject: [PATCH 020/364] libutil: Implement unix source accessors that work with file descriptors --- src/libutil/posix-source-accessor.cc | 351 +++++++++++++++++- tests/functional/fetchGit.sh | 2 +- ...val-fail-readDir-not-a-directory-1.err.exp | 2 +- ...val-fail-readDir-not-a-directory-2.err.exp | 2 +- 4 files changed, 352 insertions(+), 5 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 609855281389..bd28f81fa594 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,11 +1,19 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/util/file-system-at.hh" +#include "nix/util/lru-cache.hh" +#include "nix/util/sync.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include +#include + +#ifndef _WIN32 +# include +#endif + namespace nix { static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st) @@ -31,8 +39,14 @@ static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st namespace { +#ifndef _WIN32 + +class PosixDirectorySourceAccessor; + class PosixFileSourceAccessor : public detail::PosixSourceAccessorBase { + friend class PosixDirectorySourceAccessor; + AutoCloseFD fd; std::filesystem::path fsPath; /** @@ -122,6 +136,331 @@ std::string PosixFileSourceAccessor::readLink(const CanonPath & path) throw NotASymlink("path '%1%' is not a symlink", showPath(path)); } +static unsigned getGlobalDirFdCacheLimit() +{ + ::rlimit lim{}; + if (::getrlimit(RLIMIT_NOFILE, &lim) == -1) + throw SysError("querying RLIMIT_NOFILE"); + /* Some sane upper bound in case we have a huge rlimit. */ + return std::min(4096, lim.rlim_cur / 8); +} + +class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase +{ +public: + static unsigned getGlobalFdLimit() + { + static auto res = getGlobalDirFdCacheLimit(); + return res; + } + + static void registerAccessor(ref accessor) + { + auto reg = globalDirFdCacheRegistry.lock(); + std::erase_if(*reg, [](auto & maybeAccessor) { return maybeAccessor.expired(); }); + reg->push_back(accessor.get_ptr()); + } + +private: + AutoCloseFD dirFd; + std::filesystem::path fsPath; + + std::shared_ptr>>> dirFdCache; + + static inline std::atomic globalDirFdCount = 0; + + static inline Sync>> globalDirFdCacheRegistry; + + static void maybeEvictFromGlobalCaches() + { + if (globalDirFdCount.load(std::memory_order_relaxed) < getGlobalFdLimit()) + return; + + auto registry(globalDirFdCacheRegistry.lock()); + for (auto it = registry->begin(); it != registry->end();) { + if (globalDirFdCount.load(std::memory_order_relaxed) < getGlobalFdLimit()) + break; + + auto accessor = it->lock(); + if (!accessor) { + it = registry->erase(it); + continue; + } + + /* TODO: Would be nicer if we could evict a portion of the utilised + cache to avoid cold-start issues. Should be fine for now. */ + if (accessor->dirFdCache) { + auto cache = accessor->dirFdCache->lock(); + globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + cache->clear(); + } + + ++it; + } + } + + void insertIntoDirFdCache(const CanonPath & key, ref fd) + { + assert(dirFdCache); + auto cache = dirFdCache->lock(); + auto before = cache->size(); + cache->upsert(key, std::move(fd)); + globalDirFdCount.fetch_add(cache->size() - before, std::memory_order_relaxed); + } + + /** + * Get the parent directory of path. The second pair element might be an owning file descriptor + * if path.parent().isRoot() is false. + */ + std::pair> openParent(const CanonPath & path); + + std::function makeDirFdCallback(); + +public: + PosixDirectorySourceAccessor( + AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, unsigned dirFdCacheSize) + : PosixSourceAccessorBase(trackLastModified) + , dirFd(std::move(fd)) + , fsPath(std::move(path)) + { + assert(fsPath.is_absolute()); /* Only used for error messages, but still nice to enforce this invariant. */ + setPathDisplay(fsPath.generic_string()); + + if (dirFdCacheSize) + dirFdCache = std::make_shared>>>(dirFdCacheSize); + } + + PosixDirectorySourceAccessor(PosixDirectorySourceAccessor &&) = delete; + PosixDirectorySourceAccessor(const PosixDirectorySourceAccessor &) = delete; + PosixDirectorySourceAccessor & operator=(PosixDirectorySourceAccessor &&) = delete; + PosixDirectorySourceAccessor & operator=(const PosixDirectorySourceAccessor &) = delete; + + ~PosixDirectorySourceAccessor() + { + if (dirFdCache) { + auto cache = dirFdCache->lock(); + globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + } + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return std::filesystem::path(fsPath) / path.rel(); /* RHS *must* be a relative path. */ + } + + std::string showPath(const CanonPath & path) override + { + if (path.isRoot()) + return displayPrefix; /* No trailing slash. */ + if (displayPrefix.ends_with('/')) + return displayPrefix + path.rel(); + return displayPrefix + path.abs(); + } +}; + +std::function PosixDirectorySourceAccessor::makeDirFdCallback() +{ + if (!dirFdCache) + return nullptr; + + return [this](AutoCloseFD fd, CanonPath key) { + assert(fd); + insertIntoDirFdCache(std::move(key), make_ref(std::move(fd))); + }; +} + +std::pair> PosixDirectorySourceAccessor::openParent(const CanonPath & path) +{ + assert(!path.isRoot()); + auto parent = path.parent().value(); + if (parent.isRoot()) + return {dirFd.get(), nullptr}; + + maybeEvictFromGlobalCaches(); + + if (dirFdCache) { + if (auto cachedFd = dirFdCache->lock()->get(parent)) { + assert((*cachedFd)->get()); + return {(*cachedFd)->get(), *cachedFd}; + } + } + + AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( + dirFd.get(), parent, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, makeDirFdCallback()); + + return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; +} + +std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) +try { + PosixStat st; + + if (path.isRoot()) { + /* Must never fail - we already have the file descriptor for the directory. */ + st = nix::fstat(dirFd.get()); + } else { + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + /* We know that CanonPath returns a NUL-terminated string_view, so the use of ->data() here is safe. */ + if (::fstatat(parentFd, path.baseName()->data(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) + return std::nullopt; + throw SysError("getting status of '%1%'", showPath(path)); + } + } + + maybeUpdateMtime(st.st_mtime); + return sourceAccessorStatFromPosixStat(st); +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +try { + if (path.isRoot()) + throw NotARegularFile("'%s' is not a regular file", showPath(path)); + + AutoCloseFD fileFd = + openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + + if (!fileFd) { + if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ + throw FileNotFound("file '%s' does not exist", showPath(path)); + throw SysError("opening '%s'", showPath(path)); + } + + auto st = nix::fstat(fileFd.get()); + if (!S_ISREG(st.st_mode)) + throw Error("file '%s' has an unsupported type", showPath(path)); + PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); + maybeUpdateMtime(st.st_mtime); + fileAccessor.readFile(CanonPath::root, sink, sizeCallback); +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +try { + AutoCloseFD dirFdOwning; + + if (path.isRoot()) { + /* Get a fresh file descriptor for thread-safety. */ + dirFdOwning = ::openat(dirFd.get(), ".", O_DIRECTORY | O_RDONLY | O_CLOEXEC); + if (!dirFdOwning) + throw SysError("opening directory '%s'", showPath(path)); + } else { + dirFdOwning = openFileEnsureBeneathNoSymlinks( + dirFd.get(), path, O_DIRECTORY | O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + + if (!dirFdOwning) { + if (errno == ENOTDIR) + throw NotADirectory("'%s' is not a directory", showPath(path)); + throw SysError("opening directory '%s'", showPath(path)); + } + } + + AutoCloseDir dir(::fdopendir(dirFdOwning.get())); + if (!dir) + throw SysError("reading directory '%s'", showPath(path)); + dirFdOwning.release(); + + DirEntries entries; + const ::dirent * dirent = nullptr; + + while (errno = 0, dirent = ::readdir(dir.get())) { + checkInterrupt(); + std::string_view name(dirent->d_name); + if (name == "." || name == "..") + continue; + + std::optional type; + switch (dirent->d_type) { + case DT_REG: + type = tRegular; + break; + case DT_DIR: + type = tDirectory; + break; + case DT_LNK: + type = tSymlink; + break; + case DT_CHR: + type = tChar; + break; + case DT_BLK: + type = tBlock; + break; + case DT_FIFO: + type = tFifo; + break; + case DT_SOCK: + type = tSocket; + break; + default: + type = std::nullopt; + break; + } + entries.emplace(name, type); + } + + if (errno) + throw SysError("reading directory '%1%'", showPath(path)); + + return entries; +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +std::string PosixDirectorySourceAccessor::readLink(const CanonPath & path) +try { + if (path.isRoot()) + throw NotASymlink("file '%s' is not a symlink", showPath(path)); + + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) + throw FileNotFound("path '%s' does not exist", showPath(path)); + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + try { + return readLinkAt(parentFd, CanonPath(path.baseName().value())); + } catch (SysError & e) { + if (e.errNo == EINVAL) + throw NotASymlink("file '%s' is not a symlink", showPath(path)); + throw; + } +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +#endif + } // namespace PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) @@ -303,7 +642,7 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) ref getFSSourceAccessor() { - static auto rootFS = make_ref(); + static auto rootFS = makeFSSourceAccessor("/", /*trackLastModified=*/false); return rootFS; } @@ -383,7 +722,15 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL if (S_ISREG(st.st_mode)) return make_ref(std::move(fd), std::move(root), trackLastModified, st); - /* TODO: Use the file descriptor for fd-relative operations on the directory. */ + else if (S_ISDIR(st.st_mode)) { + auto res = make_ref( + std::move(fd), std::move(root), trackLastModified, PosixDirectorySourceAccessor::getGlobalFdLimit() / 8); + PosixDirectorySourceAccessor::registerAccessor(res); + return res; + } + + else + throw Error("file %1% has an unsupported type", PathFmt(root)); #endif return make_ref(std::move(root), trackLastModified); diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 2992020f16a8..ef57e471962d 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,7 +35,7 @@ nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit \"file://$TEST_ # Fetch a worktree. unset _NIX_FORCE_HTTP -expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath" | grepQuiet "copying '$TEST_ROOT/worktree/' to the store" +expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath" | grepQuiet "copying '$TEST_ROOT/worktree' to the store" path0=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/worktree\"; }).outPath") [[ $path0 = "$path0_" ]] diff --git a/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp b/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp index f94a7ed74521..7db863b03d53 100644 --- a/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp +++ b/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp @@ -13,4 +13,4 @@ error: | ^ 3| } - error: cannot read directory "/pwd/lang/readDir/bar": Not a directory + error: '/pwd/lang/readDir/bar' is not a directory diff --git a/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp b/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp index f5e6775545a4..20bbc3072e42 100644 --- a/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp +++ b/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp @@ -13,4 +13,4 @@ error: | ^ 3| } - error: cannot read directory "/pwd/lang/readDir/foo/git-hates-directories": Not a directory + error: '/pwd/lang/readDir/foo/git-hates-directories' is not a directory From 732f9c1cc06baf13f5b992e6ed6967da932b44dc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:47:10 +0300 Subject: [PATCH 021/364] libstore: LocalStoreAccessor uses makeFSSourceAccessor This more honestly wraps the underlying FS accessor (we want to use the unix-specific dirfd-based one). --- src/libstore/local-fs-store.cc | 52 ++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 9f5864ee4425..12818f32a447 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,4 +1,3 @@ -#include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/compression.hh" @@ -27,13 +26,14 @@ LocalFSStore::LocalFSStore(const Config & config) { } -struct LocalStoreAccessor : PosixSourceAccessor +struct LocalStoreAccessor : SourceAccessor { + ref accessor; ref store; bool requireValidPath; LocalStoreAccessor(ref store, bool requireValidPath) - : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}) + : accessor(makeFSSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})) , store(store) , requireValidPath(requireValidPath) { @@ -54,25 +54,61 @@ struct LocalStoreAccessor : PosixSourceAccessor return Stat{.type = tDirectory}; requireStoreObject(path); - return PosixSourceAccessor::maybeLstat(path); + return accessor->maybeLstat(path); + } + + Stat lstat(const CanonPath & path) override + { + /* Also allow `path` to point to the entire store, which is + needed for resolving symlinks. */ + if (path.isRoot()) + return Stat{.type = tDirectory}; + + requireStoreObject(path); + return accessor->lstat(path); } DirEntries readDirectory(const CanonPath & path) override { requireStoreObject(path); - return PosixSourceAccessor::readDirectory(path); + return accessor->readDirectory(path); } void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); - return PosixSourceAccessor::readFile(path, sink, sizeCallback); + return accessor->readFile(path, sink, sizeCallback); } std::string readLink(const CanonPath & path) override { requireStoreObject(path); - return PosixSourceAccessor::readLink(path); + return accessor->readLink(path); + } + + std::string showPath(const CanonPath & path) override + { + return accessor->showPath(path); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return accessor->getPhysicalPath(path); + } + + std::pair> getFingerprint(const CanonPath & path) override + { + return accessor->getFingerprint(path); + } + + std::optional getLastModified() override + { + return accessor->getLastModified(); + } + + bool pathExists(const CanonPath & path) override + { + return accessor->pathExists(path); } }; @@ -96,7 +132,7 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa if (!pathExists(absPath)) return nullptr; } - return std::make_shared(std::move(absPath)); + return makeFSSourceAccessor(std::move(absPath)); } const std::filesystem::path LocalFSStore::drvsLogDir = "drvs"; From 7847c5136fce2c5f7fed343b82eaebe493e5fdc6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:54:36 +0300 Subject: [PATCH 022/364] libstore: Use requireStoreObjectAccessor in addToStore --- src/libstore/local-store.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b0c41c35ce65..b41856871bd2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1077,8 +1077,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF if (info.ca) { auto & specified = *info.ca; auto actualHash = ({ - auto accessor = getFSAccessor(false); - CanonPath path{info.path.to_string()}; + SourcePath sourcePath = requireStoreObjectAccessor(info.path, /*requireValidPath=*/false); Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++ auto fim = specified.method.getFileIngestionMethod(); switch (fim) { @@ -1088,12 +1087,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF specified.hash.algo, std::string{info.path.hashPart()}, }; - dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim); + dumpPath(sourcePath, caSink, (FileSerialisationMethod) fim); h = caSink.finish().hash; break; } case FileIngestionMethod::Git: - h = git::dumpHash(specified.hash.algo, {accessor, path}).hash; + h = git::dumpHash(specified.hash.algo, sourcePath).hash; break; } ContentAddress{ From 344ab0a5ee5b6fe2147047d7c449db20095d0297 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:09:10 +0300 Subject: [PATCH 023/364] libstore: Use makeFSStoreAccessor in derivation builder With the addition of the file descriptor based source accessor (that does caching) we now create a fresh instance of the accessor and the last path component is not followed, so the transformation is safe (actually tests fail without this because some directories get unlinked and result in ENOENT if the file descriptor gets cached). --- src/libstore/unix/build/derivation-builder.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 85aa98c7ae87..2df2b74bb105 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1713,12 +1713,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() HashModuloSink caSink{outputHash.hashAlgo, oldHashPart}; auto fim = outputHash.method.getFileIngestionMethod(); dumpPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, caSink, (FileSerialisationMethod) fim); + {makeFSSourceAccessor(actualPath), CanonPath::root}, caSink, (FileSerialisationMethod) fim); return caSink.finish().hash; } case FileIngestionMethod::Git: { - return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath.native())}) - .hash; + return git::dumpHash(outputHash.hashAlgo, {makeFSSourceAccessor(actualPath), CanonPath::root}).hash; } } assert(false); @@ -1740,7 +1739,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, + {makeFSSourceAccessor(actualPath), CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.hash; @@ -1783,7 +1782,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() std::string{scratchPath->hashPart()}, std::string{requiredFinalPath.hashPart()}); rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, + {makeFSSourceAccessor(actualPath), CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0{requiredFinalPath, {store, narHashAndSize.hash}}; From b16a5c365c45fce2f3a77632bff883a100c2c5dc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:27:31 +0300 Subject: [PATCH 024/364] nix: Use makeFSSourceAccessor in place of createAtRoot The API is the same and uses the unix-specific implementation. We can also drop all makeParentCanonical calls, because the function follows symlinks in the parents already when opening the dirFd/file. One slight wrinkle it that we first have to do absPath - this is fine. --- src/nix/add-to-store.cc | 2 +- src/nix/hash.cc | 4 +--- src/nix/nix-store/nix-store.cc | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 4f0fe722f197..09972a2890af 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -36,7 +36,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!namePart) namePart = path.filename().string(); - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); + auto sourcePath = makeFSSourceAccessor(absPath(path)); auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first : store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 90f44fdecd26..9cd0592073e9 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -84,9 +84,7 @@ struct CmdHashBase : Command return std::make_unique(hashAlgo); }; - auto makeSourcePath = [&]() -> SourcePath { - return PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); - }; + auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(absPath(path)); }; Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++ switch (mode) { diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index c21625e0409c..15a4e878f5ac 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); + auto sourcePath = makeFSSourceAccessor(absPath(i)); std::cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); } } @@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); + auto sourcePath = makeFSSourceAccessor(absPath(i)); std::cout << fmt( "%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path)); } From a8bcc083a270f86334b5ef0268ddc56a23235713 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:34:47 +0300 Subject: [PATCH 025/364] libutil: Use makeFSSourceAccessor in dumpPath Unlike previously, we now follow symlinks in parents (not the last path component), but this is secure and what versions like 2.18 did. --- src/libutil/archive.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index f704acdf78b4..6188860ce13e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -101,14 +101,15 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { - auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true); + SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/true); path2.dumpPath(sink, filter); return path2.accessor->getLastModified().value(); } void dumpPath(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { - dumpPathAndGetMtime(path, sink, filter); + SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/false); + path2.dumpPath(sink, filter); } void dumpString(std::string_view s, Sink & sink) From c349fbf4c835b2eea7ab8c266f1c02d829a83029 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 02:26:06 +0300 Subject: [PATCH 026/364] libutil: Add a new overload of readDirectory (for fd-relative operations) and use in recursive traversal This ensures race-free traversal throughout. --- src/libutil/archive.cc | 34 +++++++++++-------- src/libutil/fs-sink.cc | 10 +++--- .../include/nix/util/source-accessor.hh | 16 +++++++++ src/libutil/posix-source-accessor.cc | 27 ++++++++++++++- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6188860ce13e..91af4b9e7f54 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -34,10 +34,10 @@ PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) { - auto dumpContents = [&](const CanonPath & path) { + auto dumpContents = [&sink](SourceAccessor & accessor, const CanonPath & path) { sink << "contents"; std::optional size; - readFile(path, sink, [&](uint64_t _size) { + accessor.readFile(path, sink, [&](uint64_t _size) { size = _size; sink << _size; }); @@ -47,10 +47,14 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & sink << narVersionMagic1; - [&, &this_(*this)](this const auto & dump, const CanonPath & path) -> void { + [&sink, &filter, &dumpContents]( + this const auto & dump, + SourceAccessor & accessor, + const CanonPath & path, + const CanonPath & filterPath) -> void { checkInterrupt(); - auto st = this_.lstat(path); + auto st = accessor.lstat(path); sink << "("; @@ -58,7 +62,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & sink << "type" << "regular"; if (st.isExecutable) sink << "executable" << ""; - dumpContents(path); + dumpContents(accessor, path); } else if (st.type == tDirectory) { @@ -67,7 +71,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ StringMap unhacked; - for (auto & i : this_.readDirectory(path)) + for (auto & i : accessor.readDirectory(path)) if (archiveSettings.useCaseHack) { std::string name(i.first); size_t pos = i.first.find(caseHackSuffix); @@ -81,22 +85,24 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & } else unhacked.emplace(i.first, i.first); - for (auto & i : unhacked) - if (filter((path / i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path / i.second); - sink << ")"; - } + accessor.readDirectory(path, [&](SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath) { + for (auto & i : unhacked) + if (filter((filterPath / i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second); + sink << ")"; + } + }); } else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << this_.readLink(path); + sink << "type" << "symlink" << "target" << accessor.readLink(path); else throw Error("file '%s' has an unsupported type", path); sink << ")"; - }(path); + }(*this, path, path); } time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index e41d153e92db..deb8ec3c8f7b 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -34,10 +34,12 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem } case SourceAccessor::tDirectory: { - sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) { - for (auto & [name, _] : accessor.readDirectory(from)) { - copyRecursive(accessor, from / name, dirSink, relDirPath / name); - } + sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPathTo) { + accessor.readDirectory(from, [&](SourceAccessor & subdirAccessor, const CanonPath & relDirPathFrom) { + for (auto & [name, _] : subdirAccessor.readDirectory(relDirPathFrom)) { + copyRecursive(subdirAccessor, relDirPathFrom / name, dirSink, relDirPathTo / name); + } + }); }); break; } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index c458cf8b5c35..737383aba282 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -138,6 +138,22 @@ struct SourceAccessor : std::enable_shared_from_this */ virtual DirEntries readDirectory(const CanonPath & path) = 0; + /** + * Variation of readDirectory that receives a SourceAccessor possibly scoped to \ref dirPath. + * Primary meant for recursive traversal functions that would benefit from *at-style syscalls + * relative to a particular directory. + * + * @note Like `readFile`, this method should *not* follow symlinks. + * @param callback Caller-provided function invoked with a maximally deeply scoped SourceAccessor and the path that + * would have to be prepended to each path relative to dirPath to access a particular file with it. + */ + virtual void readDirectory( + const CanonPath & dirPath, + std::function callback) + { + callback(*this, dirPath); + } + virtual std::string readLink(const CanonPath & path) = 0; virtual void dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter = defaultPathFilter); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index bd28f81fa594..6a69a16128df 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -216,6 +216,8 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase std::function makeDirFdCallback(); + AutoCloseFD openSubdirectory(const CanonPath & path); + public: PosixDirectorySourceAccessor( AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, unsigned dirFdCacheSize) @@ -249,6 +251,10 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase DirEntries readDirectory(const CanonPath & path) override; + void readDirectory( + const CanonPath & dirPath, + std::function callback) override; + std::string readLink(const CanonPath & path) override; std::optional getPhysicalPath(const CanonPath & path) override @@ -359,7 +365,7 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } -SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +AutoCloseFD PosixDirectorySourceAccessor::openSubdirectory(const CanonPath & path) try { AutoCloseFD dirFdOwning; @@ -379,6 +385,14 @@ try { } } + return dirFdOwning; +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +try { + AutoCloseFD dirFdOwning = openSubdirectory(path); AutoCloseDir dir(::fdopendir(dirFdOwning.get())); if (!dir) throw SysError("reading directory '%s'", showPath(path)); @@ -431,6 +445,17 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } +void PosixDirectorySourceAccessor::readDirectory( + const CanonPath & dirPath, + std::function callback) +{ + auto fd = openSubdirectory(dirPath); + PosixDirectorySourceAccessor accessor{ + std::move(fd), fsPath / dirPath.rel(), trackLastModified, /*dirFdCacheSize=*/0}; + callback(accessor, CanonPath::root); + PosixSourceAccessorBase::maybeUpdateMtime(accessor.mtime); +} + std::string PosixDirectorySourceAccessor::readLink(const CanonPath & path) try { if (path.isRoot()) From ba7db4e05635e351201327db0a5bbbd85ca4a07a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:38:08 +0300 Subject: [PATCH 027/364] nix-perl: Get rid of the last occurence of createAtRoot --- src/perl/lib/Nix/Store.xs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index b0f70be3a9e8..8b28b0e5397c 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath( - PosixSourceAccessor::createAtRoot(path), + makeFSSourceAccessor(absPath(path)), FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); @@ -339,7 +339,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), - PosixSourceAccessor::createAtRoot(srcPath), + makeFSSourceAccessor(absPath(srcPath)), method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { From 316e33a14834de36cd0792c021e00fddc260e3ba Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:39:14 +0300 Subject: [PATCH 028/364] libutil: Get rid of PosixSourceAccessor::createAtRoot --- .../include/nix/util/posix-source-accessor.hh | 30 ------------------- src/libutil/posix-source-accessor.cc | 10 ------- 2 files changed, 40 deletions(-) diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index d93b68e0afea..0da7dcdea858 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -76,36 +76,6 @@ public: std::optional getPhysicalPath(const CanonPath & path) override; - /** - * Create a `PosixSourceAccessor` and `SourcePath` corresponding to - * some native path. - * - * @param Whether the accessor should return a non-null getLastModified. - * When true the accessor must be used only by a single thread. - * - * The `PosixSourceAccessor` is rooted as far up the tree as - * possible, (e.g. on Windows it could scoped to a drive like - * `C:\`). This allows more `..` parent accessing to work. - * - * @note When `path` is trusted user input, canonicalize it using - * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc, - * as appropriate for the use case. At least weak canonicalization is - * required for the `SourcePath` to do anything useful at the location it - * points to. - * - * @note A canonicalizing behavior is not built in `createAtRoot` so that - * callers do not accidentally introduce symlink-related security vulnerabilities. - * Furthermore, `createAtRoot` does not know whether the file pointed to by - * `path` should be resolved if it is itself a symlink. In other words, - * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers. - * - * See - * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) - * and - * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). - */ - static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false); - void invalidateCache(const CanonPath & path) override; private: diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 6a69a16128df..affad4fbbe1f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -3,7 +3,6 @@ #include "nix/util/lru-cache.hh" #include "nix/util/sync.hh" #include "nix/util/memory-source-accessor.hh" -#include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include @@ -501,15 +500,6 @@ PosixSourceAccessor::PosixSourceAccessor() { } -SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified) -{ - std::filesystem::path path2 = absPath(path); - return { - make_ref(path2.root_path(), trackLastModified), - CanonPath{path2.relative_path().string()}, - }; -} - std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) { return root.empty() ? (std::filesystem::path{path.abs()}) From 52011de1b22206b080dc935b18a781a53417eeb0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 02:49:31 +0300 Subject: [PATCH 029/364] libstore: Use copyRecursive when copying FOD outputs This completely bypasses coroutines and serialisation/deserialisation overhead and could start using reflinking in the future too. --- src/libstore/unix/build/derivation-builder.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 2df2b74bb105..73eff4dfd6c7 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1761,8 +1761,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() any stale writable file descriptors. Copy through the serialisation/deserialisation. TODO: Use copyRecursive here and make use of reflinking. */ - auto source = sinkToSource([&](Sink & nextSink) { dumpPath(actualPath, nextSink); }); - restorePath(tmpOutput, *source, store.config->getLocalSettings().fsyncStorePaths); + auto pathAccessor = makeFSSourceAccessor(actualPath); + RestoreSink restoreSink{store.config->getLocalSettings().fsyncStorePaths}; + restoreSink.dstPath = tmpOutput; + copyRecursive(*pathAccessor, CanonPath::root, restoreSink, CanonPath::root); /* This makes it slightly harder to make sense of the control flow. The rule of thumb is that actualPath points to the current location of the stuff that we'll end up registering. */ From 8e7702d7365ef5b88b0498949d8ee2a5109f44ea Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:02:49 +0300 Subject: [PATCH 030/364] makeFSSourceAccessor: add finalSymlink parameter Useful to avoid costly resolveSymlinks() when possible. --- src/libutil/include/nix/util/source-accessor.hh | 3 ++- src/libutil/posix-source-accessor.cc | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 737383aba282..92bd5d30fae2 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -278,7 +278,8 @@ ref getFSSourceAccessor(); * * Symlinks in parents of `root` are resolved. Final symlink is not. */ -ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false); +ref makeFSSourceAccessor( + std::filesystem::path root, bool trackLastModified = false, FinalSymlink finalSymlink = FinalSymlink::DontFollow); /** * Construct an accessor that presents a "union" view of a vector of diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index affad4fbbe1f..a8046c5c1704 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -661,14 +661,14 @@ ref getFSSourceAccessor() return rootFS; } -ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified) +ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified, FinalSymlink finalSymlink) { #ifndef _WIN32 assert(root.is_absolute()); - AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); + AutoCloseFD fd = openFileReadonly(root, finalSymlink); if (!fd) { - if (errno != ELOOP) + if (finalSymlink == FinalSymlink::Follow || errno != ELOOP) throw NativeSysError("opening file %1%", PathFmt(root)); /* A helper class that holds the symlink destination in memory. */ From c296e254b133c9c6e97c47d85fb43c5ef957b7f4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:39:39 +0300 Subject: [PATCH 031/364] PosixDirectorySourceAccessor: Improve dirFd caching --- src/libutil/posix-source-accessor.cc | 37 ++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index a8046c5c1704..5dd4f56dd33e 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -293,15 +293,42 @@ std::pair> PosixDirectorySourceAccessor maybeEvictFromGlobalCaches(); + std::shared_ptr intermediateParentFd; + CanonPath anchor = CanonPath::root; + if (dirFdCache) { - if (auto cachedFd = dirFdCache->lock()->get(parent)) { - assert((*cachedFd)->get()); - return {(*cachedFd)->get(), *cachedFd}; + auto cache = dirFdCache->lock(); + auto p = parent; + while (true) { + if (auto intermediateDirFdHit = cache->get(p)) { + if (p == parent) + return {(*intermediateDirFdHit)->get(), *intermediateDirFdHit}; + intermediateParentFd = intermediateDirFdHit->get_ptr(); + anchor = p; + break; + } + if (p.isRoot()) + break; + p.pop(); + } + } + + Descriptor startFd = intermediateParentFd ? intermediateParentFd->get() : dirFd.get(); + CanonPath relPath = intermediateParentFd ? parent.removePrefix(anchor) : parent; + + std::function cb; + if (auto base = makeDirFdCallback()) { + if (intermediateParentFd) { + cb = [base = std::move(base), prefix = anchor](AutoCloseFD fd, CanonPath relKey) { + base(std::move(fd), prefix / relKey); + }; + } else { + cb = std::move(base); } } - AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( - dirFd.get(), parent, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, makeDirFdCallback()); + AutoCloseFD parentFdOwning = + openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } From 043cafab68d9e87de0e35a35f3abd8426ec5094b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:50:36 +0300 Subject: [PATCH 032/364] SourceAccessor: remove invalidateCache, remove raw usage of PosixSourceAccessor PosixSourceAccessor is going to be renamed to WindowsSourceAccessor in the following commit and hopefully deleted soon. The lstat cache is finally removed from the unix case too. --- src/libfetchers/filtering-source-accessor.cc | 5 ----- .../include/nix/fetchers/filtering-source-accessor.hh | 2 -- src/libflake/flake.cc | 2 -- src/libstore/optimise-store.cc | 8 +------- src/libutil/include/nix/util/posix-source-accessor.hh | 2 -- src/libutil/include/nix/util/source-path.hh | 5 ----- src/libutil/mounted-source-accessor.cc | 6 ------ src/libutil/posix-source-accessor.cc | 5 ----- src/libutil/union-source-accessor.cc | 6 ------ 9 files changed, 1 insertion(+), 40 deletions(-) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 6fe7d2504ec3..b8455710fa96 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -62,11 +62,6 @@ std::pair> FilteringSourceAccessor::getFin return next->getFingerprint(prefix / path); } -void FilteringSourceAccessor::invalidateCache(const CanonPath & path) -{ - next->invalidateCache(prefix / path); -} - void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 13272719fe3d..b259e8ef8f0d 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,8 +53,6 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; - /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 45871cff87ee..e8dbf42f5451 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -863,8 +863,6 @@ LockedFlake lockFlake( CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); - - flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 220e37fe9efe..15f2b1f3d137 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -153,13 +153,7 @@ void LocalStore::optimisePath_( Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = ({ - hashPath( - {make_ref(), CanonPath(path.string())}, - FileSerialisationMethod::NixArchive, - HashAlgorithm::SHA256) - .hash; - }); + Hash hash = hashPath(makeFSSourceAccessor(path), FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).hash; debug("%s has hash '%s'", PathFmt(path), hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index 0da7dcdea858..c8c54953eb81 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -76,8 +76,6 @@ public: std::optional getPhysicalPath(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; - private: /** diff --git a/src/libutil/include/nix/util/source-path.hh b/src/libutil/include/nix/util/source-path.hh index 24932e4cddbe..f15b44fdd437 100644 --- a/src/libutil/include/nix/util/source-path.hh +++ b/src/libutil/include/nix/util/source-path.hh @@ -114,11 +114,6 @@ struct SourcePath return {accessor, accessor->resolveSymlinks(path, mode)}; } - void invalidateCache() const - { - accessor->invalidateCache(path); - } - friend class std::hash; }; diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index aab95f775b77..84840352b37a 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -99,12 +99,6 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getFingerprint(subpath); } - - void invalidateCache(const CanonPath & path) override - { - auto [accessor, subpath] = resolve(path); - accessor->invalidateCache(subpath); - } }; ref makeMountedSourceAccessor(std::map> mounts) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5dd4f56dd33e..e3850d1b6db7 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -591,11 +591,6 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path return st; } -void PosixSourceAccessor::invalidateCache(const CanonPath & path) -{ - cache.erase(makeAbsPath(path).string()); -} - std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index de50c75e3733..da71903e6adf 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -90,12 +90,6 @@ struct UnionSourceAccessor : SourceAccessor } return {path, std::nullopt}; } - - void invalidateCache(const CanonPath & path) override - { - for (auto & accessor : accessors) - accessor->invalidateCache(path); - } }; ref makeUnionSourceAccessor(std::vector> && accessors) From 5aa60ea0229ec093830c33b2c05064f6fb1a2be1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:55:55 +0300 Subject: [PATCH 033/364] Rename PosixSourceAccessor to WindowsSourceAccessor This implementations sucks and really should just be removed. The assertNoSymlinks check is also useless on windows. We might as well just remove it. --- .../include/nix/util/posix-source-accessor.hh | 43 ---------- src/libutil/posix-source-accessor.cc | 86 ++++++++++++++----- 2 files changed, 63 insertions(+), 66 deletions(-) diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index c8c54953eb81..686c66471cdb 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -45,47 +45,4 @@ protected: } // namespace detail -/** - * A source accessor that uses the Unix filesystem. - */ -class PosixSourceAccessor : public detail::PosixSourceAccessorBase -{ - /** - * Optional root path to prefix all operations into the native file - * system. This allows prepending funny things like `C:\` that - * `CanonPath` intentionally doesn't support. - */ - const std::filesystem::path root; - -public: - - PosixSourceAccessor(); - PosixSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); - - void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; - - using SourceAccessor::readFile; - - bool pathExists(const CanonPath & path) override; - - std::optional maybeLstat(const CanonPath & path) override; - - DirEntries readDirectory(const CanonPath & path) override; - - std::string readLink(const CanonPath & path) override; - - std::optional getPhysicalPath(const CanonPath & path) override; - -private: - - /** - * Throw an error if `path` or any of its ancestors are symlinks. - */ - void assertNoSymlinks(CanonPath path); - - std::optional cachedLstat(const CanonPath & path); - - std::filesystem::path makeAbsPath(const CanonPath & path); -}; - } // namespace nix diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e3850d1b6db7..d5dc7be282d0 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -510,11 +510,53 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } -#endif +#else -} // namespace +/** + * A source accessor that uses the Windows filesystem. + * @todo Should be moved into a separate file. + */ +class WindowsSourceAccessor : public detail::PosixSourceAccessorBase +{ + /** + * Optional root path to prefix all operations into the native file + * system. This allows prepending funny things like `C:\` that + * `CanonPath` intentionally doesn't support. + */ + const std::filesystem::path root; + +public: + + WindowsSourceAccessor(); + WindowsSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + using SourceAccessor::readFile; -PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; + +private: + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); + + std::filesystem::path makeAbsPath(const CanonPath & path); +}; + +WindowsSourceAccessor::WindowsSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) : PosixSourceAccessorBase(trackLastModified) , root(std::move(argRoot)) { @@ -522,12 +564,12 @@ PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool displayPrefix = root.string(); } -PosixSourceAccessor::PosixSourceAccessor() - : PosixSourceAccessor(std::filesystem::path{}) +WindowsSourceAccessor::WindowsSourceAccessor() + : WindowsSourceAccessor(std::filesystem::path{}) { } -std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) +std::filesystem::path WindowsSourceAccessor::makeAbsPath(const CanonPath & path) { return root.empty() ? (std::filesystem::path{path.abs()}) : path.isRoot() ? /* Don't append a slash for the root of the accessor, since @@ -537,19 +579,13 @@ std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) : root / path.rel(); } -void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +void WindowsSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { assertNoSymlinks(path); auto ap = makeAbsPath(path); - AutoCloseFD fd = toDescriptor(open( - ap.string().c_str(), - O_RDONLY -#ifndef _WIN32 - | O_NOFOLLOW | O_CLOEXEC -#endif - )); + AutoCloseFD fd = toDescriptor(open(ap.string().c_str(), O_RDONLY)); if (!fd) throw SysError("opening file '%1%'", ap.string()); @@ -563,7 +599,7 @@ void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun>; static Cache cache; -std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +std::optional WindowsSourceAccessor::cachedLstat(const CanonPath & path) { // Note: we convert std::filesystem::path to std::string because the // former is not hashable on libc++. @@ -591,7 +627,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path return st; } -std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +std::optional WindowsSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); @@ -603,7 +639,7 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP return sourceAccessorStatFromPosixStat(*st); } -SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +SourceAccessor::DirEntries WindowsSourceAccessor::readDirectory(const CanonPath & path) { assertNoSymlinks(path); DirEntries res; @@ -655,19 +691,19 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & return res; } -std::string PosixSourceAccessor::readLink(const CanonPath & path) +std::string WindowsSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::readLink(makeAbsPath(path)).string(); } -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +std::optional WindowsSourceAccessor::getPhysicalPath(const CanonPath & path) { return makeAbsPath(path); } -void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +void WindowsSourceAccessor::assertNoSymlinks(CanonPath path) { while (!path.isRoot()) { auto st = cachedLstat(path); @@ -677,6 +713,10 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) } } +#endif + +} // namespace + ref getFSSourceAccessor() { static auto rootFS = makeFSSourceAccessor("/", /*trackLastModified=*/false); @@ -768,9 +808,9 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL else throw Error("file %1% has an unsupported type", PathFmt(root)); +#else + return make_ref(std::move(root), trackLastModified); #endif - - return make_ref(std::move(root), trackLastModified); } } // namespace nix From 5450d99984d6f000964d4589f288ee6cb37a4551 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 04:44:31 +0300 Subject: [PATCH 034/364] tests/functional/multiple-output: Move invalid outtput name tests above nuking the store We now more robustly require the existence of the store directory. These tests have just been tacked on at the end of the test, even though it nukes the store right before them. --- tests/functional/multiple-outputs.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index f703fb02be63..dec22ff817f3 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -96,6 +96,13 @@ if nix-build multiple-outputs.nix -A cyclic --no-out-link; then exit 1 fi +# TODO inspect why this doesn't work with floating content-addressing +# derivations. +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' + expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' +fi + # Do a GC. This should leave an empty store. echo "collecting garbage..." rm "$TEST_ROOT"/result* @@ -103,10 +110,3 @@ nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots rm -rf "$NIX_STORE_DIR"/.links rmdir "$NIX_STORE_DIR" - -# TODO inspect why this doesn't work with floating content-addressing -# derivations. -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' - expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' -fi From 0214ee7abbf7d0bbadbd7de9d2f6aa887328370f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 12:41:05 +0200 Subject: [PATCH 035/364] repl: Make :reload robust against load failures Previously a failing :l/:lf was recorded before evaluation, so a typo'd path or broken flake ref would be retried on every :reload. Worse, :reload cleared the file/flake lists up front and rebuilt them while iterating, so the first error aborted the loop and silently dropped all later entries (and skipped CLI installables and flakes entirely). Record loaded files/flakes only after they actually load, and during reload catch errors per entry so one broken file no longer wipes out the rest of the session. --- src/libcmd/repl.cc | 42 ++++++++++++++++++++++++++++------------ tests/functional/repl.sh | 24 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 777fc33256ca..f7833ea3a478 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -688,12 +688,13 @@ ProcessLineResult NixRepl::processLine(std::string line) void NixRepl::loadFile(const std::filesystem::path & path) { - loadedFiles.remove(path); - loadedFiles.push_back(path); Value v, v2; state->evalFile(lookupFileArg(*state, path.string()), v); state->autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); + // Remember for :reload only on success. + loadedFiles.remove(path); + loadedFiles.push_back(path); } void NixRepl::loadFlake(const std::string & flakeRefS) @@ -701,9 +702,6 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - loadedFlakes.remove(flakeRefS); - loadedFlakes.push_back(flakeRefS); - std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -730,6 +728,10 @@ void NixRepl::loadFlake(const std::string & flakeRefS) }), v); addAttrsToScope(v); + + // Remember for :reload only on success. + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); } void NixRepl::initEnv() @@ -772,28 +774,44 @@ void NixRepl::reloadFilesAndFlakes() void NixRepl::loadFiles() { - decltype(loadedFiles) old = loadedFiles; - loadedFiles.clear(); + // loadFile() rebuilds loadedFiles; keep failed entries and continue. + decltype(loadedFiles) old; + std::swap(old, loadedFiles); for (auto & i : old) { notice("Loading %1%...", PathFmt(i)); - loadFile(i); + try { + loadFile(i); + } catch (Error & e) { + loadedFiles.push_back(i); + printMsg(lvlError, e.msg()); + } } for (auto & [i, what] : getValues()) { notice("Loading installable '%1%'...", what); - addAttrsToScope(*i); + try { + addAttrsToScope(*i); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } } } void NixRepl::loadFlakes() { - Strings old = loadedFlakes; - loadedFlakes.clear(); + // See loadFiles(). + Strings old; + std::swap(old, loadedFlakes); for (auto & i : old) { notice("Loading flake '%1%'...", i); - loadFlake(i); + try { + loadFlake(i); + } catch (Error & e) { + loadedFlakes.push_back(i); + printMsg(lvlError, e.msg()); + } } } diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 9fc803d092e0..0dbc047ed827 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -281,6 +281,30 @@ exec 3>&- # Close fifo wait $repl_pid # Wait for process to finish grep -q "afterChange" repl_output +# Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, +# and an error in one loaded file must not drop later ones from the reload list. +cat > reloadA.nix < reloadB.nix < Date: Tue, 21 Apr 2026 12:49:46 +0200 Subject: [PATCH 036/364] release: name artifact jobsets maintenance-X.Y-release Suffixing the existing maintenance-X.Y identifier keeps the full-CI and release-artifact jobsets adjacent in Hydra's alphabetical list and lets tooling derive one name from the other, whereas a parallel release-X.Y namespace scatters the pair and leaves no obvious slot for master. Teach upload-release.pl to read the git revision from the legacy `src` input so it works against evaluations of the new jobset, and trim buildCross in release-jobs.nix to the two targets fallback-paths.nix actually consumes so the artifact jobset stays minimal. --- .github/workflows/upload-release.yml | 2 +- maintainers/release-process.md | 28 ++++++++++++++++------------ maintainers/upload-release.pl | 8 +++++++- packaging/release-jobs.nix | 9 ++++++++- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index cd21336c8913..f00dce4a5c6b 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: eval_id: - description: "Hydra evaluation ID" + description: "Hydra evaluation ID (from the maintenance-X.Y-release jobset)" required: true type: number is_latest: diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 19e5e050534e..aff0088a0d93 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -84,24 +84,27 @@ release: * Create two jobsets for the release branch on Hydra: `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. - `release-$VERSION` builds only the artifacts consumed by + `maintenance-$VERSION-release` builds only the artifacts consumed by `upload-release`, so a release can be cut without waiting on the full - matrix. + matrix. The `-release` suffix keeps the pair adjacent in Hydra's + alphabetical jobset list and lets scripts derive one name from the + other. * Clone the previous `maintenance-*` jobset, set identifier `maintenance-$VERSION`, description `$VERSION release branch`, flake URL `github:NixOS/nix/$VERSION-maintenance`. - * Clone the previous `release-*` jobset (or create a new **legacy** - jobset), set identifier `release-$VERSION`, description `$VERSION - release artifacts`, Nix expression `packaging/release-jobs.nix` in - input `src`, and add input `src` of type *Git checkout* pointing at + * Clone the previous `maintenance-*-release` jobset (or create a new + **legacy** jobset), set identifier `maintenance-$VERSION-release`, + description `$VERSION release artifacts`, Nix expression + `packaging/release-jobs.nix` in input `src`, and add input `src` of + type *Git checkout* pointing at `https://github.com/NixOS/nix $VERSION-maintenance`. -* Wait for the `release-$VERSION` jobset to evaluate and build. If - impatient, go to the evaluation and select `Actions -> Bump builds to - front of queue`. The aggregate job `release` turns green once every - required artifact is available. +* Wait for the `maintenance-$VERSION-release` jobset to evaluate and + build. If impatient, go to the evaluation and select `Actions -> Bump + builds to front of queue`. The aggregate job `release` turns green + once every required artifact is available. * When the release jobset evaluation has succeeded building, take note of the evaluation ID (e.g. `1780832` in @@ -177,8 +180,9 @@ release: $ git push ``` -* Wait for the desired evaluation of the `release-$VERSION` jobset to - finish building (the `release` aggregate job is the gating signal). +* Wait for the desired evaluation of the `maintenance-XX.YY-release` + jobset to finish building (the `release` aggregate job is the gating + signal). * Tag the release diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 06678553e712..b618bd900d4e 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -64,7 +64,13 @@ sub fetch { #print Dumper($evalInfo); my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; -my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; +# Flake jobsets (`maintenance-X.Y`) expose the rev via the flake URL. +# The release-artifacts jobset (`maintenance-X.Y-release`) is a legacy +# jobset whose checkout is passed in as input `src`. +my $nixRev = ($flakeInfo + ? $flakeInfo->{revision} + : $evalInfo->{jobsetevalinputs}->{src}->{revision} + // $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix-everything.x86_64-linux", 'application/json')); #print Dumper($buildInfo); diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index 23add3905e33..f82c23c9c1ac 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -8,6 +8,7 @@ # and share builds through the binary cache. # # Hydra jobset configuration: +# Identifier: maintenance--release # Type: Legacy # Nix expression: packaging/release-jobs.nix in input `src` # Inputs: @@ -36,7 +37,13 @@ let # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via # its `doc` output. build.nix-everything = hydraJobs.build.nix-everything; - buildCross.nix-everything = hydraJobs.buildCross.nix-everything; + buildCross.nix-everything = { + # Only the cross targets that end up in `fallback-paths.nix`. + inherit (hydraJobs.buildCross.nix-everything) + riscv64-unknown-linux-gnu + x86_64-unknown-freebsd + ; + }; inherit (hydraJobs) manual From 50bce0b6de7cf1e6603616c8e1ea3b14b6ed05f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 13:21:45 +0200 Subject: [PATCH 037/364] repl: Invalidate git workdir-info cache on :reload Since 7ba933e989b9, GitRepo::getCachedWorkdirInfo() memoises the workdir status (dirty flag, tracked files, head rev) in a process-global static to avoid repeated libgit2 status walks within a single command. In a long-lived `nix repl` session this cache outlives the underlying work tree: after the first `:load-flake` of a clean git checkout, every `:reload` keeps seeing the original head rev, so the source is served from the same fingerprinted store path even after the user edits files. Hook the cache invalidation into InputCache::clear(), which already expresses the "per evaluation" lifetime that resetFileCache() flushes on `:reload`, so reloading a git-backed flake (whether via `:lf` or as a CLI installable) now picks up uncommitted changes again. --- src/libfetchers/git-utils.cc | 9 +++++- .../include/nix/fetchers/git-utils.hh | 3 ++ src/libfetchers/input-cache.cc | 5 +++ tests/functional/repl.sh | 32 +++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 6bc474395af8..03a7783c1874 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1502,9 +1502,11 @@ ref Settings::getTarballCache() const } // namespace fetchers +static Sync> workdirInfoCache_; + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + auto & _cache = workdirInfoCache_; { auto cache(_cache.lock()); auto i = cache->find(path); @@ -1516,6 +1518,11 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & return workdirInfo; } +void GitRepo::invalidateWorkdirInfoCache() +{ + workdirInfoCache_.lock()->clear(); +} + bool isLegalRefName(const std::string & refName) { initLibGit2(); diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 725fbf398410..871bf5e43f56 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -88,6 +88,9 @@ struct GitRepo static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Drop all entries from the getCachedWorkdirInfo() cache. */ + static void invalidateWorkdirInfoCache(); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 85a611355e2f..3fe96d8503bc 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" @@ -65,6 +66,10 @@ struct InputCacheImpl : InputCache void clear() override { cache_.lock()->clear(); + /* The workdir info cache has the same "per evaluation" lifetime + as the input cache, so flush it here as well so that e.g. + `:reload` in `nix repl` picks up changes in git work trees. */ + GitRepo::invalidateWorkdirInfoCache(); } }; diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 0dbc047ed827..88d6e91cd90f 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -281,6 +281,38 @@ exec 3>&- # Close fifo wait $repl_pid # Wait for process to finish grep -q "afterChange" repl_output +# Regression: `:reload` on a flake loaded from a *git* work tree must pick up +# uncommitted changes. Guards against the per-process workdir-info cache +# pinning the tree to the rev seen on first load. +if [[ $(type -p git) ]]; then + createGitRepo gitflake + cat > gitflake/flake.nix <> repl_output 2>&1 & + repl_pid=$! + exec 3>repl_fifo + echo "changingThing" >&3 + for _ in $(seq 1 1000); do + grep -q "beforeChange" repl_output && break + sleep 0.1 + done + grep -q "beforeChange" repl_output || fail "git flake didn't load" + sed -i 's/beforeChange/afterChange/' gitflake/flake.nix + echo ":reload" >&3 + echo "changingThing" >&3 + echo "exit" >&3 + exec 3>&- + wait $repl_pid + grep -q "afterChange" repl_output || fail ":reload didn't pick up git work tree change" +fi + # Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, # and an error in one loaded file must not drop later ones from the reload list. cat > reloadA.nix < Date: Tue, 21 Apr 2026 12:36:37 +0200 Subject: [PATCH 038/364] PosixDirectorySourceAccessor: Drop dirfd cache on resetFileCache() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openParent() caches O_DIRECTORY fds keyed by CanonPath. When a directory at a cached path is removed and recreated, the cached fd keeps referring to the now-orphaned inode and *at() calls through it report ENOENT for entries that exist in the replacement. This shows up in `nix repl` (and any LSP / tool that keeps an EvalState alive) via the long-lived rootFS accessor: $ cat proj/default.nix { x = builtins.readFile ./sub/f; } $ echo old > proj/sub/f $ nix repl -f proj/default.nix nix-repl> x "old\n" # in another shell: rm -rf proj/sub && mkdir proj/sub && echo new > proj/sub/f # (e.g. git checkout, git stash pop, regenerating a directory) nix-repl> :r nix-repl> builtins.pathExists ./sub/f false # file is right there nix-repl> x error: path '.../proj/sub/f' does not exist Pre-#15718 nix returned "new\n" here. The evaluator already assumes the filesystem is immutable for the duration of a single evaluation, so rather than recovering lazily inside the accessor, repurpose the (now-unused) `invalidateCache()` virtual to clear the dirfd LRU and call it through `rootFS` from `EvalState::resetFileCache()` — the existing "filesystem may have changed" hook used by `:l`/`:r`/`:e` and the C API. The wrapping accessors (union, mounted, filtering) forward the call. --- src/libexpr/eval.cc | 1 + .../nix/fetchers/filtering-source-accessor.hh | 5 ++++ src/libutil-tests/source-accessor.cc | 26 +++++++++++++++++++ .../include/nix/util/source-accessor.hh | 5 ++-- src/libutil/mounted-source-accessor.cc | 5 ++++ src/libutil/posix-source-accessor.cc | 6 +++++ src/libutil/union-source-accessor.cc | 6 +++++ 7 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e4fe3ba6a5f8..fb5d8d7a5ecb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1172,6 +1172,7 @@ void EvalState::resetFileCache() fileEvalCache->clear(); inputCache->clear(); positions.clear(); + rootFS->invalidateCache(); } void EvalState::eval(Expr * e, Value & v) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index b259e8ef8f0d..c5c1ce282b30 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,6 +53,11 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; + void invalidateCache() override + { + next->invalidateCache(); + } + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index 1ee2c44cc6fc..836c5895a285 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -138,6 +138,32 @@ TEST_F(FSSourceAccessorTest, works) } } +TEST_F(FSSourceAccessorTest, invalidateCacheDropsStaleDirFds) +{ +#ifdef _WIN32 + GTEST_SKIP() << "fd-based accessor is Unix-only"; +#endif + auto accessor = makeFSSourceAccessor(tmpDir); + + createDirs(tmpDir / "a" / "b"); + writeFile(tmpDir / "a" / "b" / "f", "old"); + + EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/f"))); + + deletePath(tmpDir / "a" / "b"); + createDirs(tmpDir / "a" / "b"); + writeFile(tmpDir / "a" / "b" / "g", "new"); + createSymlink("g", tmpDir / "a" / "b" / "l"); + + accessor->invalidateCache(); + + EXPECT_FALSE(accessor->pathExists(CanonPath("a/b/f"))); + EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/g"))); + EXPECT_THAT(accessor, HasContents(CanonPath("a/b/g"), "new")); + EXPECT_THAT(accessor, HasDirectory(CanonPath("a/b"), (std::set{"g", "l"}))); + EXPECT_THAT(accessor, HasSymlink(CanonPath("a/b/l"), "g")); +} + /* ---------------------------------------------------------------------------- * RestoreSink non-directory at root (no dirFd) * --------------------------------------------------------------------------*/ diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 92bd5d30fae2..a2fbaac34993 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -231,9 +231,10 @@ struct SourceAccessor : std::enable_shared_from_this } /** - * Invalidate any cached value the accessor may have for the specified path. + * Drop any cached state that could go stale across external filesystem + * mutation (e.g. cached directory fds). */ - virtual void invalidateCache(const CanonPath & path) {} + virtual void invalidateCache() {} }; /** diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 84840352b37a..8c546bc535c6 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -73,6 +73,11 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor } } + void invalidateCache() override + { + mounts.visit_all([](auto & kv) { kv.second->invalidateCache(); }); + } + std::optional getPhysicalPath(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index d5dc7be282d0..da18d398e83f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -237,10 +237,16 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase PosixDirectorySourceAccessor & operator=(const PosixDirectorySourceAccessor &) = delete; ~PosixDirectorySourceAccessor() + { + invalidateCache(); + } + + void invalidateCache() override { if (dirFdCache) { auto cache = dirFdCache->lock(); globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + cache->clear(); } } diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index da71903e6adf..9cb004c8fd8b 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -69,6 +69,12 @@ struct UnionSourceAccessor : SourceAccessor return SourceAccessor::showPath(path); } + void invalidateCache() override + { + for (auto & accessor : accessors) + accessor->invalidateCache(); + } + std::optional getPhysicalPath(const CanonPath & path) override { for (auto & accessor : accessors) { From 387ae98fe5ad2ce8837bd56c57eecb2bc3c07aef Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:12:16 +0300 Subject: [PATCH 039/364] clang-tidy: Ban raw std::filesystem::create_directories, fix hydraJobs.clangTidy It doesn't wrap exceptions. Also clang-tidy seems to have been broken with unity builds. --- nix-meson-build-support/common/clang-tidy/.clang-tidy | 6 ++++-- packaging/hydra.nix | 2 ++ src/libfetchers/git.cc | 2 +- src/libstore-tests/register-valid-paths-bench.cc | 2 +- .../include/nix/util/tests/characterization.hh | 2 +- .../include/nix/util/tests/json-characterization.hh | 4 ++-- src/libutil/file-system.cc | 2 +- 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index f8394f4fabd4..87ce4d663b37 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -41,8 +41,6 @@ Checks: - -bugprone-unused-local-non-trivial-variable # 2 warnings - returning const& from parameter - -bugprone-return-const-ref-from-parameter - # 1 warning - unsafe C functions (e.g., getenv) - - -bugprone-unsafe-functions # 1 warning - signed char misuse - -bugprone-signed-char-misuse # 1 warning - calling parent virtual instead of override @@ -90,3 +88,7 @@ CheckOptions: bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options;__wrap___assert_fail;_SingleDerivedPathRaw;_DerivedPathRaw;_SingleBuiltPathRaw;_BuiltPathRaw' # Allow explicitly discarding return values with (void) cast bugprone-unused-return-value.AllowCastToVoid: true + bugprone-unsafe-functions.ReportDefaultFunctions: false + # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. + bugprone-unsafe-functions.CustomFunctions: > + ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 8f7e13c1f2ef..5558b0309efd 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -218,6 +218,8 @@ rec { tidyScope = pkgs.nixComponents2.overrideScope ( self: super: { withClangTidy = true; + # clang-tidy doesn't seem to like unity builds. + withUnityBuild = false; # nix-everything is built via callPackage (not the layer system), so # enableClangTidyLayer's doCheck=false doesn't reach it. Set it here # so checkInputs (the *-tests.tests.run derivations) aren't pulled in. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3941c3425660..8e959764f98b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -815,7 +815,7 @@ struct GitInputScheme : InputScheme repoDir = cacheDir; repoInfo.gitDir = "."; - std::filesystem::create_directories(cacheDir.parent_path()); + createDirs(cacheDir.parent_path()); PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true}); diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 51bcb29aa903..6417178245af 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -22,7 +22,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) auto tmpRoot = createTempDir(); auto realStoreDir = tmpRoot / "nix/store"; - std::filesystem::create_directories(realStoreDir); + createDirs(realStoreDir); std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); auto localStore = std::dynamic_pointer_cast(store); diff --git a/src/libutil-test-support/include/nix/util/tests/characterization.hh b/src/libutil-test-support/include/nix/util/tests/characterization.hh index 6dd5f38866db..154573b5e507 100644 --- a/src/libutil-test-support/include/nix/util/tests/characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/characterization.hh @@ -60,7 +60,7 @@ struct CharacterizationTest : virtual ::testing::Test auto got = test(); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile2(file, got); GTEST_SKIP() << "Updating golden master " << file; } else { diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index 9bd4d7fbf109..75eba2b06a1d 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -73,7 +73,7 @@ void checkpointJson(CharacterizationTest & test, std::string_view testStem, cons json gotJson = static_cast(got); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile(file, gotJson.dump(2) + "\n"); ADD_FAILURE() << "Updating golden master " << file; } else { @@ -98,7 +98,7 @@ void checkpointJson(CharacterizationTest & test, std::string_view testStem, cons json gotJson = static_cast(*got); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile(file, gotJson.dump(2) + "\n"); ADD_FAILURE() << "Updating golden master " << file; } else { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b7e504e13a05..0169b2728a89 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -393,7 +393,7 @@ void createDir(const std::filesystem::path & path, mode_t mode) void createDirs(const std::filesystem::path & path) { try { - std::filesystem::create_directories(path); + std::filesystem::create_directories(path); // NOLINT(bugprone-unsafe-functions) } catch (std::filesystem::filesystem_error & e) { throw SystemError(e.code(), "creating directory %1%", PathFmt(path)); } From ad15006e36eabbb91c0fb7d28655e26168d60165 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:41:39 +0300 Subject: [PATCH 040/364] Clean up dead code, redundant .string() on std::filesystem::path makeParentCanonical is no longer needed with makeFSSourceAccessor achieving exactly the same behavior without the unnecessary syscalls. Also drops leftover .string() calls from the previous migrations to std::filesystem::path. --- src/libfetchers/git.cc | 2 +- src/libstore/local-store.cc | 6 +++--- src/libutil-tests/file-system.cc | 14 -------------- src/libutil/file-system.cc | 15 --------------- src/libutil/include/nix/util/file-system.hh | 17 ----------------- src/nix/dump-path.cc | 2 +- src/nix/profile.cc | 6 +++--- 7 files changed, 8 insertions(+), 54 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8e959764f98b..447b4a3694f7 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1098,7 +1098,7 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath((*repoPath / file.rel()).string(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b41856871bd2..52fc0925f079 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1207,7 +1207,7 @@ StorePath LocalStore::addToStoreFromDump( delTempDir = std::make_unique(tempDir); tempPath = tempDir / "x"; - restorePath(tempPath.string(), bothSource, dumpMethod, localSettings.fsyncStorePaths); + restorePath(tempPath, bothSource, dumpMethod, localSettings.fsyncStorePaths); dumpBuffer.reset(); dump = {}; @@ -1260,7 +1260,7 @@ StorePath LocalStore::addToStoreFromDump( } } else { /* Move the temporary path we restored above. */ - moveFile(tempPath.string(), realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1311,7 +1311,7 @@ std::pair LocalStore::createTempDirInStore() continue; } lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); - } while (!pathExists(tmpDirFn.string()) || !lockedByUs); + } while (!pathExists(tmpDirFn) || !lockedByUs); return {tmpDirFn, std::move(tmpDirFd)}; } diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 995b75814f01..f290e1178090 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -256,20 +256,6 @@ TEST(pathExists, bogusPathDoesNotExist) ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } -/* ---------------------------------------------------------------------------- - * makeParentCanonical - * --------------------------------------------------------------------------*/ - -TEST(makeParentCanonical, noParent) -{ - ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file"))); -} - -TEST(makeParentCanonical, root) -{ - ASSERT_EQ(makeParentCanonical(FS_ROOT), FS_ROOT_NO_TRAILING_SLASH); -} - /* ---------------------------------------------------------------------------- * chmodIfNeeded * --------------------------------------------------------------------------*/ diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 0169b2728a89..20dbf3ff3d61 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -660,21 +660,6 @@ bool isExecutableFileAmbient(const std::filesystem::path & exe) == 0; } -std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) -{ - std::filesystem::path path(absPath(rawPath)); - try { - auto parent = path.parent_path(); - if (parent == path) { - // `path` is a root directory => trivially canonical - return parent; - } - return std::filesystem::canonical(parent) / path.filename(); - } catch (std::filesystem::filesystem_error & e) { - throw SystemError(e.code(), "canonicalising parent path of %1%", PathFmt(path)); - } -} - void chmod(const std::filesystem::path & path, mode_t mode) { if ( diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index f977e089bfef..2a13b311c9b1 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -159,23 +159,6 @@ std::optional maybeStat(const std::filesystem::path & path); */ bool pathExists(const std::filesystem::path & path); -/** - * Canonicalize a path except for the last component. - * - * This is useful for getting the canonical location of a symlink. - * - * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will - * resolve the symlink `l` to its target. - * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target, - * but does ensure that the returned parent part of the path, `foo` is resolved - * to `canonical("foo")`, and can therefore be retrieved without traversing any - * symlinks. - * - * If a relative path is passed, it will be made absolute, so that the parent - * can always be canonicalized. - */ -std::filesystem::path makeParentCanonical(const std::filesystem::path & path); - /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 25129487786b..f21374c62337 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,7 +61,7 @@ struct CmdDumpPath2 : Command void run() override { auto sink = getNarSink(); - dumpPath(path.string(), sink); + dumpPath(path, sink); sink.flush(); } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c85c406aa518..e63d0150c08e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -125,7 +125,7 @@ struct ProfileManifest auto manifestPath = profile / "manifest.json"; if (std::filesystem::exists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath.string())); + auto json = nlohmann::json::parse(readFile(manifestPath)); auto version = json.value("version", 0); std::string sUrl; @@ -248,13 +248,13 @@ struct ProfileManifest } } - buildProfile(tempDir.string(), std::move(pkgs)); + buildProfile(tempDir, std::move(pkgs)); writeFile(tempDir / "manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; - dumpPath(tempDir.string(), sink); + dumpPath(tempDir, sink); auto narHash = hashString(HashAlgorithm::SHA256, sink.s); From 3e56d682f8e20c635f435af4ec91fdc5e8f661e2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:52:49 +0300 Subject: [PATCH 041/364] Fix FreeBSD build, return includes, now with IWYU comments Got needlessly broken by trimming includes: https://hydra.nixos.org/build/326607395/nixlog/1 --- src/libexpr-test-support/tests/value/context.cc | 4 +--- src/libexpr-tests/derived-path.cc | 4 +--- src/libstore-test-support/derived-path.cc | 5 +---- .../include/nix/store/tests/outputs-spec.hh | 2 +- src/libstore-test-support/path.cc | 4 +--- src/libutil-test-support/hash.cc | 4 +--- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index ca7996acc16e..22f8aa7cf0ff 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/expr/tests/value/context.hh" diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index c685f6a094a8..a67f3df77b96 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,8 +1,6 @@ #include #include -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index ee8018c3268f..c27edc95ef9b 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,7 +1,4 @@ - -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index a30f83770257..6cdb0a60ef96 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/outputs-spec.hh" diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 98a255ccc026..bca404cde455 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index d9c7a0f74798..2dc5da5f2c14 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/util/hash.hh" From 61e1be2c6d1fd8bef7186959b85ea950b8aacf61 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:54:52 +0300 Subject: [PATCH 042/364] Fix Darwin build Also needlessly broken by trimming includes... https://hydra.nixos.org/build/326734471/nixlog/1 --- src/libstore/optimise-store.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 15f2b1f3d137..eeac67ad27f3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -7,6 +7,10 @@ #include #include +#ifdef __APPLE__ +# include +#endif + #include #include #include From b74401e8714151ef0e961ca7cd13bb7e2c9cc316 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 22:07:57 +0300 Subject: [PATCH 043/364] Fix functional_root tests This check isn't reliable with CAP_DAC_OVERRIDE. It's also not very critical anyway. --- tests/functional/flakes/edit.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/flakes/edit.sh b/tests/functional/flakes/edit.sh index 0d926c28745d..2758c397ec88 100755 --- a/tests/functional/flakes/edit.sh +++ b/tests/functional/flakes/edit.sh @@ -9,4 +9,3 @@ nix edit "$flake1Dir#" | grepQuiet simple.builder.sh tar --exclude=".git*" -czf "$TEST_ROOT"/flake1Dir.tar.gz -C "$(dirname "$flake1Dir")" "$(basename "$flake1Dir")" # Test that editing a file from a tarball flake works and the file is readonly. nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" | grepQuiet simple.builder.sh -EDITOR='test ! -w' nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" From d1b5ac384a22a8aeae29f54ea1898596c82fc172 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 23:35:07 +0300 Subject: [PATCH 044/364] Move std::filesystem cleanup Fix several issues: * Get rid of raw usage of std::filesystem::remove where it would only be removing links - not directories. std::filesystem::remove is equivalent to POSIX remove(3) and it there unlink is the better choice. * Get rid of more unnecessary .string() usage. * Replace exists(symlink_status()) with pathExists - it has the same semantics of not following symlinks. * Replace std::filesystem::remove_all usage with deletePath - mostly in tests to prepare to ban it via clang-tidy - C++ stdlib doesn't make any guarantees about symlink race safety for those. --- src/libfetchers/git-utils.cc | 4 ++-- .../include/nix/store/tests/nix_api_store.hh | 11 ++++------- src/libstore-tests/register-valid-paths-bench.cc | 2 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/optimise-store.cc | 6 +++--- src/libstore/profiles.cc | 11 +---------- src/libutil-tests/unix/file-system-at.cc | 14 ++++---------- src/libutil/file-system.cc | 8 ++++++++ src/libutil/include/nix/util/file-system.hh | 7 +++++++ 10 files changed, 32 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 03a7783c1874..216dcb741e94 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -226,7 +226,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options) { - if (pathExists(path.string())) + if (pathExists(path)) return; if (!options.create) @@ -570,7 +570,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile.string())) + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index 15df329cb2b1..df48a7469ea8 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -22,13 +22,10 @@ public: }; ~nix_api_store_test_base() override - { - if (exists(std::filesystem::path{nixDir})) { - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); - } - std::filesystem::remove_all(nixDir); - } + try { + nix::deletePath(nixDir); + } catch (...) { + nix::ignoreExceptionInDestructor(); } std::string nixDir; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 6417178245af..ecea1c8010a4 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -66,7 +66,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) state.PauseTiming(); localStore.reset(); store.reset(); - std::filesystem::remove_all(tmpRoot); + deletePath(tmpRoot); state.ResumeTiming(); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index afbd47b5bd60..8d1a16f91281 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -262,7 +262,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists((config->realStoreDir.get() / storePath.to_string()).string()); + return pathExists(config->realStoreDir.get() / storePath.to_string()); }; bool errors = false; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 52fc0925f079..75f485c31f22 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1365,7 +1365,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printError( "link %s was modified! expected hash %s, got '%s'", PathFmt(link.path()), name.string(), hash); if (repair) { - std::filesystem::remove(link.path()); + unlinkIfExists(link.path()); printInfo("removed link %s", PathFmt(link.path())); } else { errors = true; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index eeac67ad27f3..0b85c6e3dc20 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -164,7 +164,7 @@ void LocalStore::optimisePath_( std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ - if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size || (repair && hash != ({ hashPath( @@ -178,11 +178,11 @@ void LocalStore::optimisePath_( warn( "There may be more corrupted paths." "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); - std::filesystem::remove(linkPath); + unlinkIfExists(linkPath); } } - if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ try { std::filesystem::create_hard_link(path, linkPath); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 519c2abc9805..d015548e5e65 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -95,19 +95,10 @@ std::filesystem::path createGeneration(LocalFSStore & store, std::filesystem::pa return generation; } -static void removeFile(const std::filesystem::path & path) -{ - try { - std::filesystem::remove(path); - } catch (std::filesystem::filesystem_error & e) { - throw SystemError(e.code(), "removing file %1%", PathFmt(path)); - } -} - void deleteGeneration(const std::filesystem::path & profile, GenerationNumber gen) { std::filesystem::path generation = makeName(profile, gen); - removeFile(generation); + unlinkIfExists(generation); } /** diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 09e427a26faf..e453a80d325b 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -42,8 +42,6 @@ TEST(fchmodatTryNoFollow, works) auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); ASSERT_TRUE(dirFd); - struct ::stat st; - using nix::testing::ThrowsSysError; /* Check that symlinks are not followed and targets are not changed. @@ -82,22 +80,18 @@ TEST(fchmodatTryNoFollow, works) }; expectSymlinkChmod("filelink", 0777); - ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0644); + ASSERT_EQ(stat(tmpDir / "file").st_mode & 0777, 0644); expectSymlinkChmod("dirlink", 0777); - ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0755); + ASSERT_EQ(stat(tmpDir / "dir").st_mode & 0777, 0755); /* Check fchmodatTryNoFollow works on regular files and directories. */ EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("file"), 0600)); - ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0600); + ASSERT_EQ(stat(tmpDir / "file").st_mode & 0777, 0600); EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700)); - ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0700); + ASSERT_EQ(stat(tmpDir / "dir").st_mode & 0777, 0700); EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath("nonexistent"), 0600); }, ThrowsSysError(ENOENT)); } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 20dbf3ff3d61..b3087700d733 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -679,6 +679,14 @@ void chmod(const std::filesystem::path & path, mode_t mode) # define UNLINK_PROC ::unlink #endif +void unlinkIfExists(const std::filesystem::path & path) +{ + if (UNLINK_PROC(path.c_str()) == -1) { + if (errno != ENOENT) + throw SysError("removing %s", PathFmt(path)); + } +} + void unlink(const std::filesystem::path & path) { if (UNLINK_PROC(path.c_str()) == -1) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 2a13b311c9b1..fca79ceb8ba6 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -514,6 +514,13 @@ void chown(const std::filesystem::path & path, uid_t owner, gid_t group); */ void unlink(const std::filesystem::path & path); +/** + * Remove a file, throwing an exception on error. ENOENT is ignored. + * + * @param path Path to the file to remove. + */ +void unlinkIfExists(const std::filesystem::path & path); + /** * Try to remove a file, ignoring errors. * From b3e84b77679e727d4cba0b54c49ba0588b387093 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 23:42:18 +0300 Subject: [PATCH 045/364] Ban std::filesystem::remove_all in the codebase All instances of those must use deletePath instead - standard library implementation is not guaranteed to be robust against symlink races. --- nix-meson-build-support/common/clang-tidy/.clang-tidy | 1 + src/libutil/windows/file-system.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index 87ce4d663b37..daf4c7f7c4b8 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -92,3 +92,4 @@ CheckOptions: # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. bugprone-unsafe-functions.CustomFunctions: > ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; + ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 98d41a9caa25..2ac2f74f80c1 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -77,7 +77,7 @@ std::filesystem::path defaultTempDir() void deletePath(const std::filesystem::path & path) { std::error_code ec; - std::filesystem::remove_all(path, ec); + std::filesystem::remove_all(path, ec); // NOLINT(bugprone-unsafe-functions) if (ec && ec != std::errc::no_such_file_or_directory) throw SysError(ec.default_error_condition().value(), "recursively deleting %1%", PathFmt(path)); } From 77db3d45cd36386ee222730419ad15c68adf4d4b Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:25:24 +0000 Subject: [PATCH 046/364] libutil: Bound NAR directory depth and guard coroutine stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse()/dumpPath() recurse per directory level with no bound. On the main stack this overflows at a few thousand levels (DoS). Inside a sinkToSource/sourceToSink coroutine the stack is a 128 KiB malloc() chunk with no guard page, so overflow writes into adjacent heap — the v<1.25 wopAddToStore handler runs parseDump() on exactly such a stack and any client can downgrade to that protocol version. Reject NARs deeper than narMaxDepth (64) in both parse() and dumpPath(), and allocate coroutine stacks with boost::coroutines2::protected_fixedsize_stack so any remaining overflow hits a guard page instead of the heap. --- src/libutil/archive.cc | 25 +++++++++++++++++++------ src/libutil/serialise.cc | 5 +++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 91af4b9e7f54..cfc0a510704f 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -30,6 +30,12 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); +/* Maximum directory nesting depth for dumpPath()/parseDump(). Bounds + stack usage so deep trees cannot overflow the (possibly coroutine) + stack these run on. Chosen to fit comfortably in the default 128 KiB + boost coroutine stack. */ +static constexpr size_t narMaxDepth = 64; + PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) @@ -51,9 +57,13 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & this const auto & dump, SourceAccessor & accessor, const CanonPath & path, - const CanonPath & filterPath) -> void { + const CanonPath & filterPath, + size_t depth) -> void { checkInterrupt(); + if (depth >= narMaxDepth) + throw Error("path '%s' exceeds maximum NAR directory depth of %d", accessor.showPath(path), narMaxDepth); + auto st = accessor.lstat(path); sink << "("; @@ -89,7 +99,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & for (auto & i : unhacked) if (filter((filterPath / i.first).abs())) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second); + dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second, depth + 1); sink << ")"; } }); @@ -102,7 +112,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & throw Error("file '%s' has an unsupported type", path); sink << ")"; - }(*this, path, path); + }(*this, path, path, 0); } time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) @@ -153,8 +163,11 @@ struct CaseInsensitiveCompare } }; -static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path) +static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path, size_t depth) { + if (depth >= narMaxDepth) + throw badArchive("NAR directory nesting exceeds maximum depth of %d", narMaxDepth); + auto getString = [&]() { checkInterrupt(); return readString(source); @@ -237,7 +250,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("node"); - parse(dirSink, source, relDirPath / name); + parse(dirSink, source, relDirPath / name, depth + 1); expectTag(")"); } @@ -268,7 +281,7 @@ void parseDump(FileSystemObjectSink & sink, Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, CanonPath::root); + parse(sink, source, CanonPath::root, 0); } void restorePath(const std::filesystem::path & path, Source & source, bool startFsync) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6c77c15fe584..542cee2b0278 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -10,6 +10,7 @@ #include #include +#include #ifdef _WIN32 # include @@ -328,7 +329,7 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = coro_t::push_type([&](coro_t::pull_type & yield) { + coro = coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -385,7 +386,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type([&](coro_t::push_type & yield) { + coro = coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) { yield(data); From 0d461dc28e9624685b3cc09ab877e2d3976c2b69 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:37:51 +0000 Subject: [PATCH 047/364] libutil: Bound string lengths in the NAR parser readString() defaults to no length limit, so every token, directory entry name, and symlink target in a NAR was read into a freshly allocated std::string of attacker-chosen size before any validation. A multi-GB length prefix where "(" is expected would be allocated and filled before expectTag() rejects it. Cap tags and keywords at 32 bytes, entry names at 255, and symlink targets at 4095. Overlong strings now throw SerialisationError("string is too long") without allocating. --- src/libutil/archive.cc | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index cfc0a510704f..3ddc280f9519 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -168,33 +168,42 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath if (depth >= narMaxDepth) throw badArchive("NAR directory nesting exceeds maximum depth of %d", narMaxDepth); - auto getString = [&]() { + /* NAR keywords are all <= 10 bytes; a little slack keeps error + messages useful for short garbage without allowing large + allocations. */ + constexpr size_t narMaxTag = 32; + /* Format-defined bounds, intentionally independent of host + NAME_MAX/PATH_MAX. */ + constexpr size_t narMaxName = 255; + constexpr size_t narMaxTarget = 4095; + + auto getString = [&](size_t max) { checkInterrupt(); - return readString(source); + return readString(source, max); }; auto expectTag = [&](std::string_view expected) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag != expected) - throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024)); + throw badArchive("expected tag '%s', got '%s'", expected, tag); }; expectTag("("); expectTag("type"); - auto type = getString(); + auto type = getString(narMaxTag); if (type == "regular") { sink.createRegularFile(path, [&](auto & crf) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag == "executable") { - auto s2 = getString(); + auto s2 = getString(0); if (s2 != "") throw badArchive("executable marker has non-empty value"); crf.isExecutable(); - tag = getString(); + tag = getString(narMaxTag); } if (tag != "contents") @@ -213,7 +222,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath std::string prevName; while (1) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag == ")") break; @@ -225,7 +234,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("name"); - auto name = getString(); + auto name = getString(narMaxName); if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) throw badArchive("NAR contains invalid file name '%1%'", name); @@ -260,7 +269,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath else if (type == "symlink") { expectTag("target"); - auto target = getString(); + auto target = getString(narMaxTarget); sink.createSymlink(path, target); expectTag(")"); From ee68e870ece0332cf7886950ae9892296b36b821 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:37:51 +0000 Subject: [PATCH 048/364] libutil: Reject empty and NUL-containing symlink targets in NARs A NUL byte in a symlink target truncates at the symlink(2) boundary, so "foo\0junk" and "foo" restore to identical filesystem state but have different NAR hashes. LocalStore::addToStore() hashes the incoming stream, while `nix store verify` and re-export hash a fresh dump of the on-disk tree, so a non-canonical NAR would pass the ingest check but fail every later verification. Rejecting NUL (and empty, which symlink(2) refuses anyway) keeps parse() injective on accepted inputs, which the narHash model depends on. --- src/libutil/archive.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3ddc280f9519..cd7f58263c7d 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -270,6 +270,8 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("target"); auto target = getString(narMaxTarget); + if (target.empty() || target.find((char) 0) != std::string::npos) + throw badArchive("NAR contains invalid symlink target"); sink.createSymlink(path, target); expectTag(")"); From 5d8406285b7d973477037135f3226fef0fb74db5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 01:04:03 +0300 Subject: [PATCH 049/364] daemon: Limit the number of crashes before exiting to 64 In case someone is intentionally crashing the daemon and brute-force ASLR or get some other kind of info leak about the address-space layout, we'd like to to limit the blast radius by dying and (maybe) being restarted to get a fresh address space layout. --- .../include/nix/cmd/unix-socket-server.hh | 3 ++ src/libcmd/unix/unix-socket-server.cc | 3 ++ src/nix/unix/daemon.cc | 35 +++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa79317..48202cf18293 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -68,6 +68,8 @@ struct ServeUnixSocketOptions #endif }; +MakeError(AbortServeSocket, BaseError); + /** * Run a server loop that accepts connections and calls the handler for each. * @@ -83,6 +85,7 @@ struct ServeUnixSocketOptions * * This function never returns normally. It runs until interrupted * (e.g., via SIGINT), at which point it throws `Interrupted`. + * Can be explicitly exited by throwing AbortServeSocket. * * @param options Configuration for the server. * @param handler Callback invoked for each accepted connection. diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba462207..c0348fa3b444 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -122,6 +122,9 @@ PeerInfo getPeerInfo(Descriptor remote) handler(std::move(remote), [&]() { listeningSockets.clear(); }); } + } catch (AbortServeSocket &) { + /* Explicitly aborted, bail out. */ + throw; } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 05e47f79c36b..4bc7a512d9e4 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -280,6 +280,21 @@ static void daemonLoop( } #endif + /* Check for anything that might be a crash. Too many crashes aren't + supposed to happen and we should limit the amount if someone is + intentionally triggering those as an ASLR bypass attempt (each forked + daemon worker has the same address space layout as we do). TODO: Ideally + we'd re-exec the daemon worker so that it gets a fresh address space + for each connection. Alternatively, we could make the daemon socket use + Accept=yes systemd.socket(5). */ + unsigned crashCount = 0; + + /* For now we are just limiting the number of crashes experienced by this + daemon instance. systemd (e.g.) would restart us, which would get us + a fresh address space layout - which is exactly what we want in case + someone is intentionally crashing the daemon to brute-force ASLR. */ + static constexpr unsigned crashLimit = 64; + try { unix::serveUnixSocket( { @@ -287,13 +302,29 @@ static void daemonLoop( .socketMode = 0666, .auxiliaryFd = sigChldPipe.pipe.readSide.get(), .onAuxiliaryFdPollin = - []() { + [&crashCount]() { sigChldPipe.drain(); /* Reap all dead children. */ pid_t pid = -1; int status; - while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) + while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) { printInfo("reaped child process %1%, status = %2%", pid, statusToString(status)); + + if (!WIFSIGNALED(status)) + continue; + + int sig = WTERMSIG(status); + for (auto i : {SIGILL, SIGSEGV, SIGBUS, SIGABRT, SIGSYS, SIGFPE}) { + if (sig == i) { + printInfo("daemon worker %1% crashed", pid); + ++crashCount; + break; + } + } + + if (crashCount >= crashLimit) + throw unix::AbortServeSocket("too many daemon worker crashes (%1%)", crashLimit); + } }, }, [&](AutoCloseFD remote, std::function closeListeners) { From 88b5bcd07751de3b5ca174d5192cf4d160434748 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 01:06:24 +0300 Subject: [PATCH 050/364] libutil: Make formatter happy --- src/libutil/serialise.cc | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 542cee2b0278..4e6daa84c9b4 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -329,20 +329,21 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { - LambdaSource source([&](char * out, size_t out_len) { - if (cur.empty()) { - yield(); - if (yield.get()) - throw EndOfFile("coroutine has finished"); - } - - size_t n = cur.copy(out, out_len); - cur.remove_prefix(n); - return n; + coro = + coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { + LambdaSource source([&](char * out, size_t out_len) { + if (cur.empty()) { + yield(); + if (yield.get()) + throw EndOfFile("coroutine has finished"); + } + + size_t n = cur.copy(out, out_len); + cur.remove_prefix(n); + return n; + }); + reader(source); }); - reader(source); - }); } if (!*coro) { @@ -386,14 +387,15 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { - LambdaSink sink([&](std::string_view data) { - if (!data.empty()) { - yield(data); - } + coro = + coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { + LambdaSink sink([&](std::string_view data) { + if (!data.empty()) { + yield(data); + } + }); + writer(sink); }); - writer(sink); - }); } if (cur.empty()) { From 7cdf3bad750b2af7c91e973f86ab8833312fa7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 23 Apr 2026 00:23:28 +0200 Subject: [PATCH 051/364] packaging: build the Rust nix-installer with embedded Nix Builds NixOS/nix-installer via buildRustPackage with this revision's Nix closure baked in, exposed as hydraJobs.rustInstaller.. Replaces the dropped --nix-package-url knob (#15728). Source pinned via fetchFromGitHub to avoid a circular flake input. --- flake.nix | 3 ++ packaging/hydra.nix | 26 +++++++++++ packaging/rust-installer/default.nix | 64 ++++++++++++++++++++++++++++ packaging/rust-installer/tarball.nix | 50 ++++++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 packaging/rust-installer/default.nix create mode 100644 packaging/rust-installer/tarball.nix diff --git a/flake.nix b/flake.nix index 89cbe93cff7f..c9be5e1dd58c 100644 --- a/flake.nix +++ b/flake.nix @@ -466,6 +466,9 @@ ) ) ) + // lib.optionalAttrs (self.hydraJobs.rustInstaller ? ${system}) { + rustInstaller = self.hydraJobs.rustInstaller.${system}; + } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 5558b0309efd..e3f8f1c1f1cb 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -298,6 +298,32 @@ rec { } ); + # `NixOS/nix-installer` with this revision's Nix closure embedded. + rustInstaller = + lib.genAttrs + ( + linux64BitSystems + ++ [ + "x86_64-darwin" + "aarch64-darwin" + ] + ) + ( + system: + let + pkgs = nixpkgsFor.${system}.native; + # Embed the native (glibc) Nix even though the Linux installer + # binary is static/musl. + tarball = pkgs.callPackage ./rust-installer/tarball.nix { + nix = pkgs.nixComponents2.nix-everything; + }; + builder = if pkgs.stdenv.hostPlatform.isLinux then pkgs.pkgsStatic else pkgs; + in + builder.callPackage ./rust-installer { + inherit tarball; + } + ); + # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix new file mode 100644 index 000000000000..dcfbf4badc31 --- /dev/null +++ b/packaging/rust-installer/default.nix @@ -0,0 +1,64 @@ +# `NixOS/nix-installer` built with *this* Nix closure embedded, so +# Hydra/CI can dogfood the Rust installer without the (removed) +# `--nix-package-url` knob. +{ + lib, + stdenv, + rustPlatform, + fetchFromGitHub, + tarball, +}: + +let + installerVersion = "2.34.5"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "nix-installer"; + tag = installerVersion; + hash = "sha256-+gM241qQOzQlOnP0a7d47z3iRf9+yNjbBJCLIWWNX+c="; + }; +in + +rustPlatform.buildRustPackage { + pname = "nix-installer"; + version = tarball.passthru.nixVersion; + + inherit src; + + cargoHash = "sha256-6pt2f7wznH672L5+SkbA5GA6Sxvk1KamAf3erGqZlLU="; + + doCheck = false; + + env = { + NIX_TARBALL_PATH = "${tarball}/nix.tar.zst"; + NIX_STORE_PATH = tarball.passthru.nixStorePath; + NSS_CACERT_STORE_PATH = tarball.passthru.cacertStorePath; + NIX_VERSION = tarball.passthru.nixVersion; + } + // lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + ''; + + # The binary embeds store-path strings (`NIX_STORE_PATH`, …) on + # purpose; don't let the reference scanner pull the whole Nix + # closure into this derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; +} diff --git a/packaging/rust-installer/tarball.nix b/packaging/rust-installer/tarball.nix new file mode 100644 index 000000000000..4f236e0215bb --- /dev/null +++ b/packaging/rust-installer/tarball.nix @@ -0,0 +1,50 @@ +# Zstd-compressed Nix closure in the layout expected by +# `NixOS/nix-installer` (`include_bytes!` at build time). +{ + lib, + stdenv, + runCommand, + buildPackages, + zstd, + nix, + cacert, +}: + +let + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ + nix + cacert + ]; + }; +in + +runCommand "nix-installer-tarball-${nix.version}" + { + nativeBuildInputs = [ zstd ]; + + passthru = { + nixStorePath = nix.outPath; + cacertStorePath = cacert.outPath; + nixVersion = nix.version; + }; + } + '' + mkdir -p $out + + dir=nix-${nix.version}-${stdenv.hostPlatform.system} + + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + + tar cf - \ + --sort=name \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) \ + | zstd -19 -T1 -o $out/nix.tar.zst + '' From b12be692c8c1f73ea13bde46c00100fbe4a851c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 23 Apr 2026 00:23:28 +0200 Subject: [PATCH 052/364] ci: run the Rust installer in installer_test Build .#rustInstaller alongside the bash installer, ship it in the same artifact and run the binary directly; drop the now-dead experimental-installer plumbing from install-nix-action. --- .../actions/install-nix-action/action.yaml | 70 ------------------- .github/workflows/ci.yml | 33 ++++----- .../prepare-installer-for-github-actions | 6 +- 3 files changed, 20 insertions(+), 89 deletions(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index 535ae9d08fd9..f889944b2d03 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -4,22 +4,12 @@ inputs: dogfood: description: "Whether to use Nix installed from the latest artifact from master branch" required: true # Be explicit about the fact that we are using unreleased artifacts - experimental-installer: - description: "Whether to use the experimental installer to install Nix" - default: false - experimental-installer-version: - description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used." - # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one - default: "latest" extra_nix_config: description: "Gets appended to `/etc/nix/nix.conf` if passed." install_url: description: "URL of the Nix installer" required: false default: "https://releases.nixos.org/nix/nix-2.32.1/install" - tarball_url: - description: "URL of the Nix tarball to use with the experimental installer" - required: false github_token: description: "Github token" required: true @@ -51,74 +41,14 @@ runs: gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" env: GH_TOKEN: ${{ inputs.github_token }} DOGFOOD_REPO: "NixOS/nix" - - name: "Gather system info for experimental installer" - shell: bash - if: ${{ inputs.experimental-installer == 'true' }} - run: | - echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)" - - if [ "$RUNNER_OS" == "Linux" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="linux" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - elif [ "$RUNNER_OS" == "macOS" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="darwin" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - if [ "$RUNNER_ARCH" == "X64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=x86_64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - elif [ "$RUNNER_ARCH" == "ARM64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=aarch64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH" - exit 1 - fi - - echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - env: - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - name: "Download latest experimental installer" - shell: bash - id: download-latest-experimental-installer - if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }} - run: | - RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId") - - EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT" - mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - # Executable permissions are lost in artifacts - find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} + - echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ inputs.github_token }} - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 - if: ${{ inputs.experimental-installer != 'true' }} with: # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} extra_nix_config: ${{ inputs.extra_nix_config }} - - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20 - if: ${{ inputs.experimental-installer == 'true' }} - with: - diagnostic-endpoint: "" - # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/`? - local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }} - source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} - nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} - extra-conf: ${{ inputs.extra_nix_config }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be8cb0f0c4fd..07329d16a202 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,19 +164,19 @@ jobs: - scenario: on ubuntu runs-on: ubuntu-24.04 os: linux - experimental-installer: false + rust-installer: false - scenario: on macos runs-on: macos-14 os: darwin - experimental-installer: false - - scenario: on ubuntu (experimental) + rust-installer: false + - scenario: on ubuntu (rust) runs-on: ubuntu-24.04 os: linux - experimental-installer: true - - scenario: on macos (experimental) + rust-installer: true + - scenario: on macos (rust) runs-on: macos-14 os: darwin - experimental-installer: true + rust-installer: true name: installer test ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} steps: @@ -188,22 +188,19 @@ jobs: path: out - name: Looking up the installer tarball URL id: installer-tarball-url - run: | - echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" + run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 - if: ${{ !matrix.experimental-installer }} + if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - uses: ./.github/actions/install-nix-action - if: ${{ matrix.experimental-installer }} - with: - dogfood: false - experimental-installer: true - tarball_url: ${{ steps.installer-tarball-url.outputs.tarball-path }} - github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Run rust installer + if: ${{ matrix.rust-installer }} + run: | + chmod +x out/nix-installer + ./out/nix-installer install --no-confirm + env: + RUST_BACKTRACE: full - run: sudo apt install fish zsh if: matrix.os == 'linux' - run: brew install fish diff --git a/ci/gha/tests/prepare-installer-for-github-actions b/ci/gha/tests/prepare-installer-for-github-actions index 0fbecf25c2aa..e240e56fca0a 100755 --- a/ci/gha/tests/prepare-installer-for-github-actions +++ b/ci/gha/tests/prepare-installer-for-github-actions @@ -2,10 +2,14 @@ set -euo pipefail -nix build -L ".#installerScriptForGHA" ".#binaryTarball" +nix build -L \ + ".#installerScriptForGHA" \ + ".#binaryTarball" \ + ".#rustInstaller" mkdir -p out cp ./result/install "out/install" name="$(basename "$(realpath ./result-1)")" # everything before the first dash cp -r ./result-1 "out/${name%%-*}" +cp ./result-2/bin/nix-installer "out/nix-installer" From 23d53de80a631bc2f5fefe2ed1d2f5b865755bc6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 02:09:52 +0300 Subject: [PATCH 053/364] PosixDirectorySourceAccessor: Correct the SymlinkNotAllowed error message with cached parent dirfds This wouldn't display the prefix that's already cached. --- src/libutil/posix-source-accessor.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index da18d398e83f..1fe3bf134666 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -333,10 +333,14 @@ std::pair> PosixDirectorySourceAccessor } } - AutoCloseFD parentFdOwning = - openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); - - return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; + try { + AutoCloseFD parentFdOwning = + openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); + return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; + } catch (SymlinkNotAllowed & e) { + /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ + throw SymlinkNotAllowed(anchor / e.path, "path '%s' is a symlink", showPath(anchor / e.path)); + } } std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) From 3e458a7af4544925624d7b00513571e178d78c5a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 02:35:06 +0300 Subject: [PATCH 054/364] PosixDirectorySourceAccessor: Improve dirFd caching for readFile/openSubdirectory Those didn't do caching without openat2 and did a bit more syscalls that ideal. With openat2 this is a slight regression, but overall not too meaningful. --- src/libutil/posix-source-accessor.cc | 93 +++++++++++++++++----------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 1fe3bf134666..661922cd39cc 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -207,6 +207,13 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase globalDirFdCount.fetch_add(cache->size() - before, std::memory_order_relaxed); } + /** + * Helper for opening the parent directory used for other FD-relative operations down the line. + * Also caches the resulting dirFd. Throws FileNotFound if some intermediate directory or the parent + * doesn't exist. + */ + std::pair> openParentAndUpsert(const CanonPath & path, bool ignoreMissing); + /** * Get the parent directory of path. The second pair element might be an owning file descriptor * if path.parent().isRoot() is false. @@ -343,25 +350,39 @@ std::pair> PosixDirectorySourceAccessor } } +std::pair> +PosixDirectorySourceAccessor::openParentAndUpsert(const CanonPath & path, bool ignoreMissing) +{ + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ { + if (ignoreMissing) + return {INVALID_DESCRIPTOR, {}}; + else + throw FileNotFound("path '%s' does not exist", showPath(path)); + } + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + return {parentFd, parentFdOwning}; +} + std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) -try { +{ PosixStat st; if (path.isRoot()) { /* Must never fail - we already have the file descriptor for the directory. */ st = nix::fstat(dirFd.get()); } else { - auto [parentFd, parentFdOwning] = openParent(path); - if (parentFd == INVALID_DESCRIPTOR) { - if (errno == ENOENT || errno == ENOTDIR) - return std::nullopt; - throw SysError("opening directory '%1%'", showPath(path.parent().value())); - } - - if (dirFdCache && parentFdOwning) { - assert(*parentFdOwning); - insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); - } + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/true); + if (parentFd == INVALID_DESCRIPTOR) + return std::nullopt; /* We know that CanonPath returns a NUL-terminated string_view, so the use of ->data() here is safe. */ if (::fstatat(parentFd, path.baseName()->data(), &st, AT_SYMLINK_NOFOLLOW) == -1) { @@ -373,20 +394,26 @@ try { maybeUpdateMtime(st.st_mtime); return sourceAccessorStatFromPosixStat(st); -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) -try { +{ if (path.isRoot()) throw NotARegularFile("'%s' is not a regular file", showPath(path)); - AutoCloseFD fileFd = - openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + /* TODO: We can do better when we have openat2. */ + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); + AutoCloseFD fileFd; + + try { + fileFd = openFileEnsureBeneathNoSymlinks(parentFd, path.baseName().value(), O_RDONLY | O_CLOEXEC); + } catch (SymlinkNotAllowed & e) { + auto parent = path.parent().value(); + throw SymlinkNotAllowed(parent / e.path, "path '%s' is a symlink", showPath(parent / e.path)); + } if (!fileFd) { - if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ + if (errno == ENOENT) throw FileNotFound("file '%s' does not exist", showPath(path)); throw SysError("opening '%s'", showPath(path)); } @@ -397,12 +424,10 @@ try { PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); maybeUpdateMtime(st.st_mtime); fileAccessor.readFile(CanonPath::root, sink, sizeCallback); -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } AutoCloseFD PosixDirectorySourceAccessor::openSubdirectory(const CanonPath & path) -try { +{ AutoCloseFD dirFdOwning; if (path.isRoot()) { @@ -411,8 +436,16 @@ try { if (!dirFdOwning) throw SysError("opening directory '%s'", showPath(path)); } else { - dirFdOwning = openFileEnsureBeneathNoSymlinks( - dirFd.get(), path, O_DIRECTORY | O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + /* TODO: We can do better when we have openat2. */ + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); + + try { + dirFdOwning = + openFileEnsureBeneathNoSymlinks(parentFd, path.baseName().value(), O_DIRECTORY | O_RDONLY | O_CLOEXEC); + } catch (SymlinkNotAllowed & e) { + auto parent = path.parent().value(); + throw SymlinkNotAllowed(parent / e.path, "path '%s' is a symlink", showPath(parent / e.path)); + } if (!dirFdOwning) { if (errno == ENOTDIR) @@ -422,8 +455,6 @@ try { } return dirFdOwning; -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) @@ -497,17 +528,7 @@ try { if (path.isRoot()) throw NotASymlink("file '%s' is not a symlink", showPath(path)); - auto [parentFd, parentFdOwning] = openParent(path); - if (parentFd == INVALID_DESCRIPTOR) { - if (errno == ENOENT || errno == ENOTDIR) - throw FileNotFound("path '%s' does not exist", showPath(path)); - throw SysError("opening directory '%1%'", showPath(path.parent().value())); - } - - if (dirFdCache && parentFdOwning) { - assert(*parentFdOwning); - insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); - } + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); try { return readLinkAt(parentFd, CanonPath(path.baseName().value())); From 794f00a5776c03160147a3a58df5abffd9d9f2ee Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 03:01:40 +0300 Subject: [PATCH 055/364] Add CachingSourceAccessor for the sake of the evaluator, cache positive lstat/readlinks This avoids the footgun and issues around cache invalidation, while also keeping lookups more efficient. The evaluator already assumes a mostly immutable view of the filesystem (and we do nuke the caches during :r repl command). We used to cache lstat results because we do symlink resolution manually. Also adds the readDirectory override to the LocalStoreAccessor. --- src/libexpr/eval.cc | 2 + src/libstore/local-fs-store.cc | 8 ++ src/libutil/caching-source-accessor.cc | 102 ++++++++++++++++++ .../include/nix/util/source-accessor.hh | 6 ++ src/libutil/meson.build | 1 + 5 files changed, 119 insertions(+) create mode 100644 src/libutil/caching-source-accessor.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fb5d8d7a5ecb..7eb27f45b87d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -277,6 +277,8 @@ EvalState::EvalState( mounted fetchTree. */ auto accessor = settings.pureEval ? storeFS.cast() : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); + /* Cache positive lstat/readlink results to speed up resolveSymlinks. */ + accessor = makeCachingSourceAccessor(accessor); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 12818f32a447..3fc724f5fe74 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -74,6 +74,14 @@ struct LocalStoreAccessor : SourceAccessor return accessor->readDirectory(path); } + void readDirectory( + const CanonPath & dirPath, + std::function callback) override + { + requireStoreObject(dirPath); + return accessor->readDirectory(dirPath, std::move(callback)); + } + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); diff --git a/src/libutil/caching-source-accessor.cc b/src/libutil/caching-source-accessor.cc new file mode 100644 index 000000000000..041fab0d621a --- /dev/null +++ b/src/libutil/caching-source-accessor.cc @@ -0,0 +1,102 @@ +#include "nix/util/source-accessor.hh" + +#include + +namespace nix { + +class CachingSourceAccessor : public SourceAccessor +{ + ref next; + + boost::concurrent_flat_map lstatCache; + boost::concurrent_flat_map readLinkCache; + +public: + CachingSourceAccessor(ref next_) + : next(std::move(next_)) + { + displayPrefix.clear(); + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override + { + next->readFile(path, sink, sizeCallback); + } + + std::optional maybeLstat(const CanonPath & path) override + { + if (auto res = getConcurrent(lstatCache, path)) + return *res; + + auto st = next->maybeLstat(path); + if (!st) + return std::nullopt; + + /* Never evict, the evaluator better keep positive lookups cached. */ + lstatCache.emplace(path, *st); + return st; + } + + Stat lstat(const CanonPath & path) override + { + if (auto res = getConcurrent(lstatCache, path)) + return *res; + + auto st = next->lstat(path); + /* Never evict, the evaluator better keep positive lookups cached. */ + lstatCache.emplace(path, st); + return st; + } + + DirEntries readDirectory(const CanonPath & path) override + { + return next->readDirectory(path); + } + + void readDirectory( + const CanonPath & dirPath, + std::function callback) override + { + return next->readDirectory(dirPath, std::move(callback)); + } + + std::string readLink(const CanonPath & path) override + { + if (auto res = getConcurrent(readLinkCache, path)) + return *res; + + auto target = next->readLink(path); + /* Never evict, the evaluator better keep positive lookups cached. */ + readLinkCache.emplace(path, target); + return target; + } + + std::string showPath(const CanonPath & path) override + { + return next->showPath(path); + } + + void invalidateCache() override + { + lstatCache.clear(); + readLinkCache.clear(); + next->invalidateCache(); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return next->getPhysicalPath(path); + } + + std::pair> getFingerprint(const CanonPath & path) override + { + return next->getFingerprint(path); + } +}; + +ref makeCachingSourceAccessor(ref next) +{ + return make_ref(std::move(next)); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index a2fbaac34993..cc076054e73e 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -288,4 +288,10 @@ ref makeFSSourceAccessor( */ ref makeUnionSourceAccessor(std::vector> && accessors); +/** + * Make a wrapper source accessor that caches positive lookup results. + * Useful for the evaluator which already assumes a mostly immutable view of the filesystem. + */ +ref makeCachingSourceAccessor(ref next); + } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 50ade6688726..d132ce67c748 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -136,6 +136,7 @@ sources = [ config_priv_h ] + files( 'base-n.cc', 'base-nix-32.cc', 'bump-memory-resource.cc', + 'caching-source-accessor.cc', 'canon-path.cc', 'compression-algo.cc', 'compression-settings.cc', From 6c2f1b446c9f6d0acf09703d28ba10b895de0c6c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 22:48:01 +0300 Subject: [PATCH 056/364] libexpr: Cache findFile lookups better We were doing a bunch of pointless I/O (even before dirfd accessor) for non-existent paths in nixPath for stuff like and repeated lookups. This caches positive and negative lookups and nukes the cache in resetFileCache for the purposes of repl. --- src/libexpr/eval.cc | 34 ++++++++++++++++++++++------ src/libexpr/include/nix/expr/eval.hh | 15 +++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7eb27f45b87d..700d9a1ebe6b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1173,6 +1173,7 @@ void EvalState::resetFileCache() importResolutionCache->clear(); fileEvalCache->clear(); inputCache->clear(); + lookupPathResolved->clear(); positions.clear(); rootFS->invalidateCache(); } @@ -3269,14 +3270,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ continue; auto r = *rOpt; - auto res = (r / CanonPath(suffix)).resolveSymlinks(); - if (res.pathExists()) + auto suffixPath = CanonPath(suffix); + if (auto cachedRes = getConcurrent(*rOpt->resolvedPaths, suffixPath)) { + if (*cachedRes) + return **cachedRes; + else + // Cached negative lookup. + continue; + } + + auto res = (r.path / suffixPath).resolveSymlinks(); + if (res.pathExists()) { + r.resolvedPaths->emplace(suffixPath, res); return res; + } // Backward compatibility hack: throw an exception if access // to this path is not allowed. if (auto accessor = res.accessor.dynamic_pointer_cast()) accessor->checkAccess(res.path); + + // Cache negative lookups too. + r.resolvedPaths->emplace(suffixPath, std::nullopt); } if (hasPrefix(path, "nix/")) @@ -3290,17 +3305,22 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ .debugThrow(); } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::shared_ptr +EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; if (auto cached = getConcurrent(*lookupPathResolved, value)) return *cached; - auto finish = [&](std::optional res) { - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else + auto finish = [&](std::optional maybePath) { + std::shared_ptr res; + if (maybePath) { + debug("resolved search path element '%s' to '%s'", value, *maybePath); + res = std::make_shared( + *maybePath, make_ref()); + } else { debug("failed to resolve search path element '%s'", value); + } lookupPathResolved->emplace(std::string(value), res); return res; }; diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 89e2d5099da2..20765f85546b 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -489,7 +489,15 @@ private: LookupPath lookupPath; - const ref, StringViewHash, std::equal_to<>>> + struct LookupPathResolvedState + { + SourcePath path; + const ref>> resolvedPaths; + }; + + const ref< + boost:: + concurrent_flat_map, StringViewHash, std::equal_to<>>> lookupPathResolved; /** @@ -626,9 +634,10 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt`. + * If it is not found, return `nullptr`. */ - std::optional resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); + std::shared_ptr + resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); /** * Evaluate an expression to normal form From bae48a5ba2ac21d4936f2a8d68aba9c603746a00 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2026 15:47:56 +0200 Subject: [PATCH 057/364] Input::getAccessorUnchecked(): Wrap fetches in a path lock This prevents multiple processes (like nix-eval-jobs instances) from fetching the same input at the same time. That doesn't matter for correctness, but it can cause a lot of redundant downloads. --- src/libfetchers/fetchers.cc | 16 ++++++++++++++++ tests/functional/tarball.sh | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index fb87f9b94506..c47b07c43b47 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -7,6 +7,9 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/url.hh" +#include "nix/util/users.hh" +#include "nix/store/pathlocks.hh" +#include "nix/util/environment-variables.hh" #include @@ -301,6 +304,19 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + std::filesystem::create_directories(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. We only do this for final diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index e7d9b96fda33..6deb7b96b7c6 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -116,3 +116,17 @@ path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" # Test that unpacking an empty file does not segfault (see https://github.com/NixOS/nix/issues/15116). touch "$TEST_ROOT/empty" expectStderr 1 nix store prefetch-file --unpack "file://$TEST_ROOT/empty" | grepQuiet "archive.*is empty" + +# Test that concurrent invocations of Nix will fetch the tarball only once. +rm -rf "$TEST_HOME/.cache" +store="$TEST_ROOT/prefetch-store" +nix-store --store "$store" --init # needed because concurrent creation of the store can give SQLite errors +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log1" & +pid1="$!" +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log2" & +pid2="$!" +wait "$pid1" +wait "$pid2" +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "Download.*to") -eq 2 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "downloading.*tar.tar") -eq 1 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "waiting for another Nix process to finish fetching input") -eq 1 ]] From eb5c6775512df79d557687fe259f9223fbc5e348 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Apr 2026 23:35:26 +0300 Subject: [PATCH 058/364] libfetchers: Acquire fetcher-locks only when input is not substituted When substituting an input, the store in question already acquires internal PathLocks on the to-be-substituted store path. Acquiring a lock eagerly is pessimising the case where multiple stores are substituted to concurrenty (but with the same cache dir). Also changes std::filesystem::create_directories to createDirs which wraps exceptions nicely and adds a setDeletion() call to clean up lock files. --- src/libfetchers/fetchers.cc | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c47b07c43b47..4eedafa22a24 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -304,19 +304,6 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up - * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ - auto lockFilePath = - getCacheDir() / "fetcher-locks" - / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); - std::filesystem::create_directories(lockFilePath.parent_path()); - PathLocks lock( - {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); - - static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; - if (inTest) - std::this_thread::sleep_for(std::chrono::seconds(1)); - /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. We only do this for final @@ -358,6 +345,21 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings } } + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. Note that we don't need + * this when substituting the input. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + createDirs(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + lock.setDeletion(true); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto [accessor, result] = scheme->getAccessor(settings, store, *this); if (!accessor->fingerprint) From 5f5fb7a3a538525200505d65762d44bc6df7ffad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Apr 2026 18:14:10 +0200 Subject: [PATCH 059/364] RemoteStore::addToStore(): Fix version comparison --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2def1bf87aa7..88c3847bb3cf 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -460,7 +460,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair void RemoteStore::addMultipleToStore( PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) { - if (getConnection()->protoVersion < WorkerProto::Version{.number = {1, 32}}) { + if (getConnection()->protoVersion.number < WorkerProto::Version::Number{1, 32}) { Store::addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs); return; } From f63e1383143ac15e4e47b975c629f4298a8d9e0a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 24 Apr 2026 22:51:07 +0300 Subject: [PATCH 060/364] Tune error messages --- src/libutil/posix-source-accessor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 661922cd39cc..b5921381d313 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -346,7 +346,7 @@ std::pair> PosixDirectorySourceAccessor return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } catch (SymlinkNotAllowed & e) { /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ - throw SymlinkNotAllowed(anchor / e.path, "path '%s' is a symlink", showPath(anchor / e.path)); + throw SymlinkNotAllowed(anchor / e.path, "path '%s' (or its ancestor) is a symlink", showPath(anchor / e.path)); } } @@ -420,7 +420,7 @@ void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, auto st = nix::fstat(fileFd.get()); if (!S_ISREG(st.st_mode)) - throw Error("file '%s' has an unsupported type", showPath(path)); + throw NotARegularFile("file '%s' is not a regular file", showPath(path)); PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); maybeUpdateMtime(st.st_mtime); fileAccessor.readFile(CanonPath::root, sink, sizeCallback); From fb4d488dd55119703411aced25385a7346791ff2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 14:26:17 +0300 Subject: [PATCH 061/364] libflake: Drop unused NixStringContext in getFlake This was some leftovers from detnix cherry-pick, we ban string contexts in getFlake and have always. --- src/libflake/flake-primops.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 3ad66611726b..1fc4f7ccbde9 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -42,7 +42,6 @@ PrimOp getFlake(const Settings & settings) auto path = state.realisePath(pos, *args[0]); callFlake(state, lockFlake(settings, state, path, lockFlags), v); } else { - NixStringContext context; std::string flakeRefS( state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); From 569ee752c341db5de5fa63962d01f6a5d4cb995e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:20:34 +0300 Subject: [PATCH 062/364] libexpr: Add a way to collect string context from ValuePrinter --- src/libexpr/include/nix/expr/print.hh | 13 +++++++++++-- src/libexpr/print.cc | 15 ++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/libexpr/include/nix/expr/print.hh b/src/libexpr/include/nix/expr/print.hh index 229f7159d15a..8e6d0f9bf09a 100644 --- a/src/libexpr/include/nix/expr/print.hh +++ b/src/libexpr/include/nix/expr/print.hh @@ -10,6 +10,7 @@ #include #include "nix/util/fmt.hh" +#include "nix/expr/value/context.hh" #include "nix/expr/print-options.hh" namespace nix { @@ -64,7 +65,12 @@ bool isReservedKeyword(const std::string_view str); */ std::ostream & printIdentifier(std::ostream & o, std::string_view s); -void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions{}); +void printValue( + EvalState & state, + std::ostream & str, + Value & v, + PrintOptions options = PrintOptions{}, + NixStringContext * context = nullptr); /** * A partially-applied form of `printValue` which can be formatted using `<<` @@ -77,12 +83,15 @@ private: EvalState & state; Value & value; PrintOptions options; + NixStringContext * context; public: - ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions{}) + ValuePrinter( + EvalState & state, Value & value, PrintOptions options = PrintOptions{}, NixStringContext * context = nullptr) : state(state) , value(value) , options(options) + , context(context) { } }; diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index f2f62a636982..0c95ae60c5b6 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -162,6 +162,7 @@ class Printer std::ostream & output; EvalState & state; PrintOptions options; + NixStringContext * context; std::optional seen; size_t totalAttrsPrinted = 0; size_t totalListItemsPrinted = 0; @@ -577,9 +578,12 @@ class Printer printBool(v); break; - case nString: + case nString: { printString(v); + if (context) + copyContext(v, *context); break; + } case nPath: printPath(v); @@ -632,10 +636,11 @@ class Printer } public: - Printer(std::ostream & output, EvalState & state, PrintOptions options) + Printer(std::ostream & output, EvalState & state, PrintOptions options, NixStringContext * context) : output(output) , state(state) , options(options) + , context(context) { } @@ -656,14 +661,14 @@ class Printer } }; -void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options) +void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options, NixStringContext * context) { - Printer(output, state, options).print(v); + Printer(output, state, options, context).print(v); } std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) { - printValue(printer.state, output, printer.value, printer.options); + printValue(printer.state, output, printer.value, printer.options, printer.context); return output; } From a8e2e0022732edc2602ad6849dd333213def2621 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Sat, 25 Apr 2026 13:26:37 -0700 Subject: [PATCH 063/364] linux-derivation-builder: Also block *listxattr Not blocking *listxattr not only leaks information from the host filesystem (e.g. leaks the fact that SELinux is enabled through the presence of security.selinux in the xattrs) but can confuse applications that assume that if *listxattr succeeds, xattrs are supported. For example, mkfs.ubifs will call llistxattr to get a list of attributes with a check for errno == EOPNOTSUPP to detect xattrs being unsupported. If that succeeds and it finds xattrs (which it will on an SELinux system), it calls lgetxattr on the individual attributes, which fails, causing mkfs.ubifs to fail. --- src/libstore/unix/build/linux-derivation-builder.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0bc32a91e648..1f636ec56f2f 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -116,7 +116,10 @@ static void setupSeccomp(const LocalSettings & localSettings) /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(listxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(llistxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(flistxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 From 891ef140b8564a7848a3d75976e172c3e15ec14b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:46:58 +0300 Subject: [PATCH 064/364] Don't copy flakes to the store unnecessarily This builds on top of https://github.com/NixOS/nix/pull/14050 to actually make flakes get lazily copied to the store. This repurposes a slightly less lazy (but also more deterministic) approach than determinate nix has taken. We do still pay to cost of hashing an input once to compute the store path and narHash daemon-client-side. This could be improved in follow-ups in case we don't actually need to check the narHash (like during local development). We need certain backwards compatibility hacks for getFlake with a discarded string context, those are similar to what detnix does. See: https://github.com/DeterminateSystems/nix-src/pull/422 See: https://github.com/DeterminateSystems/nix-src/pull/402 Co-authored-by: Eelco Dolstra --- src/libcmd/installable-value.cc | 5 +- src/libexpr/include/nix/expr/eval.hh | 27 ++++++++++- .../include/nix/expr/print-ambiguous.hh | 8 +++- src/libexpr/paths.cc | 47 ++++++++++++++++++- src/libexpr/primops.cc | 36 ++++++++++---- src/libexpr/print-ambiguous.cc | 14 ++++-- src/libflake/flake-primops.cc | 17 +++++++ src/nix/app.cc | 2 + src/nix/eval.cc | 16 ++++--- src/nix/flake.cc | 11 +++-- src/nix/nix-instantiate/nix-instantiate.cc | 4 +- tests/functional/flakes/flakes.sh | 1 - 12 files changed, 159 insertions(+), 29 deletions(-) diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 3a167af3db49..92811c1d01da 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -54,8 +54,11 @@ InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std:: } else if (v.type() == nString) { + auto path = state->coerceToSingleDerivedPath(pos, v, errorCtx); + if (auto o = std::get_if(&path.raw())) + state->ensureLazyPathCopied(o->path); return {{ - .path = DerivedPath::fromSingle(state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .path = DerivedPath::fromSingle(path), .info = make_ref(), }}; } diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 20765f85546b..1283b27cf822 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -738,6 +738,26 @@ public: std::optional tryAttrsToString( const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + enum class CopyLazyPaths : bool { + PreserveLazy = false, + Copy = true, + }; + + /** + * For efficiency reasons, some store paths (as seen by the evaluator) in + * the storeFS at their content-addressed locations don't get copied to the + * store eagerly. This saves on needless I/O and possibly IPC if all the + * evaluator does is just evaluate nix expressions from those locations. + * This function copies such store objects to the store if they aren't already valid. + */ + void ensureLazyPathCopied(const StorePath & path); + + /** + * Ensure that all NixStringContextElem::Opaque context elements get fetched + * to the store. + */ + void ensureLazyPathsCopied(const NixStringContext & context); + /** * String coercion. * @@ -1044,9 +1064,14 @@ public: /** * Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`. + * @param copyLazyPaths When encountering a lazy path (i.e. a string with Opaque context that's also "mounted" on + * the storeFS), fetch the store path to the store. */ SourcePath realisePath( - const PosIdx pos, Value & v, std::optional resolveSymlinks = SymlinkResolution::Full); + const PosIdx pos, + Value & v, + std::optional resolveSymlinks = SymlinkResolution::Full, + CopyLazyPaths copyLazyPaths = CopyLazyPaths::PreserveLazy); /** * Realise the given string with context, and return the string with outputs instead of downstream output diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index 7e44a6b66ebc..07fcc337e355 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -17,6 +17,12 @@ class EvalState; * * See: https://github.com/NixOS/nix/issues/9730 */ -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth = 0); +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context = nullptr, + size_t depth = 0); } // namespace nix diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index ca303208173e..ff09b7aaa5e0 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -22,10 +22,55 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +void EvalState::ensureLazyPathCopied(const StorePath & path) +{ + if (settings.isReadOnly()) + return; + + auto mount = storeFS->getMount(CanonPath(store->printStorePath(path))); + if (!mount) + return; + + /* TODO: We could memoise this in-memory if necessary. */ + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + /* Force a copy. mountInput does a dryRun to just calculate the storePath and narHash. */ + FetchMode::Copy, + path.name()); + + /* Catch hash mismatches more loudly. This is more likely caused by unsound + caching of different accessor types that fetch the same repo with + the same git revision, but with different kinds of accessors (think + tarball-based fetchers vs local/remote git accessors). */ + if (storePath != path) { + panic(fmt( + "hashed store path computed by the evaluator ('%1%') does not match what was computed when copying to the store ('%2%'), this is a bug", + store->printStorePath(path), + store->printStorePath(storePath))); + } +} + +void EvalState::ensureLazyPathsCopied(const NixStringContext & context) +{ + for (const auto & c : context) + if (auto * o = std::get_if(&c.raw)) + /* TODO: This could be done in parallel. */ + ensureLazyPathCopied(o->path); +} + StorePath EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) { - auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + /* To mount the input, dryRun is sufficient. We still compute the narHash (to check for mismatches) and the store + path to figure out where to mount it. TODO: This could be relaxed in the future by making outPath and narHash + lazier. Good code that doesn't do `toString ./.` or otherwise inspects the outPath string and only uses it for + doing relative imports does not even require computing the store path. That is a big invasive change though and + would require having a special "LazyStorePathString" thunk. narHash also doesn't need to be computed eagerly in + case it's not actually specified (like during local development with a dirty tree) - in that case narHash could + also become a lazy app/thunk that shares the state with the storePath delayed computation. */ + auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1ff6..0c472fb10217 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "nix/store/names.hh" #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/util/util.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" @@ -63,7 +64,7 @@ std::string EvalState::realiseString(Value & s, StorePathSet * storePathsOutMayb nix::NixStringContext stringContext; auto rawStr = coerceToString(pos, s, stringContext, "while realising a string").toOwned(); auto rewrites = realiseContext(stringContext, storePathsOutMaybe, isIFD); - + ensureLazyPathsCopied(stringContext); return nix::rewriteStrings(rawStr, rewrites); } @@ -88,7 +89,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + /* If the path happens to be mounted on the storeFS, that means it's lazy path string and would get + copied to the store on-demand (when referenced in a derivation). The string is equal to final + store path where the store object would end up (the path is hashed before mounting). */ + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -158,7 +163,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS return res; } -SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional resolveSymlinks) +SourcePath EvalState::realisePath( + const PosIdx pos, Value & v, std::optional resolveSymlinks, CopyLazyPaths copyLazyPaths) { NixStringContext context; @@ -167,6 +173,8 @@ SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional(&c.raw)) + if (auto p = std::get_if(&c.raw)) { + state.ensureLazyPathCopied(p->path); refs.insert(p->path); - else + } else state .error( "files created by %1% may not reference derivations, but %2% references %3%", diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ed91cad85a47..b0a5224f26e6 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -7,7 +7,13 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth) +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context, + size_t depth) { checkInterrupt(); @@ -22,6 +28,8 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setlexicographicOrder(state.symbols)) { str << state.symbols[i->name] << " = "; - printAmbiguous(state, *i->value, str, seen, depth + 1); + printAmbiguous(state, *i->value, str, seen, context, depth + 1); str << "; "; } str << "}"; @@ -52,7 +60,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setisInStore(sourcePath->string())) { + auto [storePath, subPath] = state.store->toStorePath(sourcePath->string()); + if (auto mount = state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + auto path = state.storePath(storePath) / CanonPath(subPath); + if (!flakeRef.subdir.empty()) + path = path / flakeRef.subdir; + return callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } + } + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); } }; diff --git a/src/nix/app.cc b/src/nix/app.cc index 634db04f3fe1..167ffd8d86f8 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -95,6 +95,8 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) c.raw)); } + state.ensureLazyPathsCopied(context); + return UnresolvedApp{App{ .context = std::move(context2), .program = program, diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ed6fe41be655..6b8c0ba12234 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -83,10 +83,10 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption [&](this const auto & recurse, Value & v, const PosIdx pos, const std::filesystem::path & path) -> void { state->forceValue(v, pos); - if (v.type() == nString) - // FIXME: disallow strings with contexts? + if (v.type() == nString) { + copyContext(v, context); writeFile(path, v.string_view()); - else if (v.type() == nAttrs) { + } else if (v.type() == nAttrs) { [[maybe_unused]] bool directoryCreated = std::filesystem::create_directory(path); // Directory should not already exist assert(directoryCreated); @@ -110,9 +110,8 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (raw) { logger->stop(); - writeFull( - getStandardOutput(), - *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + auto string = state->coerceToString(noPos, *v, context, "while generating the eval command output"); + writeFull(getStandardOutput(), *string); } else if (json) { @@ -120,8 +119,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } else { - logger->cout("%s", ValuePrinter(*state, *v, PrintOptions{.force = true, .derivationPaths = true})); + ValuePrinter printer(*state, *v, PrintOptions{.force = true, .derivationPaths = true}, &context); + logger->cout("%s", printer); } + + state->ensureLazyPathsCopied(context); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2a5239549e0f..53719f5ec7f7 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "nix/expr/get-drvs.hh" #include "nix/util/os-string.hh" #include "nix/util/signals.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/store/store-open.hh" #include "nix/store/derivations.hh" #include "nix/store/outputs-spec.hh" @@ -215,8 +216,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; - // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + /* Flakes do not get copied to the store, but are instead mounted at + their expected store paths in storeFS. Querying metadata does not + force copying to the store, as one would expect. */ + auto storePath = store->toStorePath(flake.path.path.abs()).first; if (json) { nlohmann::json j; @@ -238,7 +241,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = storePath; + j["path"] = store->printStorePath(storePath); j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); @@ -249,7 +252,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout(ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", flake.lockedRef.to_string()); if (flake.description) logger->cout(ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); - logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", storePath); + logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", store->printStorePath(storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout(ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index 27a11767b004..dee79bcbfc2c 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -66,7 +66,7 @@ void processExpr( if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(state, vRes, std::cout, &seen); + printAmbiguous(state, vRes, std::cout, &seen, &context); std::cout << std::endl; } } else { @@ -94,6 +94,8 @@ void processExpr( std::cout << fmt("%s%s\n", drvPathS, (outputName != "out" ? "!" + outputName : "")); } } + + state.ensureLazyPathsCopied(context); } } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 8d95f61240bd..fec80cc89529 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -69,7 +69,6 @@ nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' # Test 'nix flake metadata --json'. json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] -[[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) [[ -n $(echo "$json" | jq -r .fingerprint) ]] From d4ea1a07d2c14042e9e2cb359fa4efb39a244127 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 23 Apr 2026 22:21:48 -0400 Subject: [PATCH 065/364] Enable more clang-tidy checks, related fixes Signed-off-by: Lisanna Dettwyler --- .../common/clang-tidy/.clang-tidy | 17 +++++++++++------ src/libexpr-tests/nix_api_expr.cc | 4 ++-- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval-error.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/get-drvs.cc | 6 +++--- src/libexpr/include/nix/expr/diagnose.hh | 4 ++-- src/libexpr/include/nix/expr/nixexpr.hh | 2 +- src/libexpr/include/nix/expr/parser-state.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/build/derivation-goal.cc | 4 ++++ src/libstore/build/goal.cc | 4 ++-- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/worker.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/build/goal.hh | 4 ++-- src/libstore/include/nix/store/sqlite.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/sqlite.cc | 2 +- src/libstore/store-api.cc | 18 ++++++++---------- src/libstore/unix/build/child.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 4 ++-- src/libutil-tests/file-descriptor.cc | 4 ++-- src/libutil/file-descriptor.cc | 1 + src/libutil/include/nix/util/async.hh | 4 ++-- src/libutil/include/nix/util/closure.hh | 2 +- src/libutil/include/nix/util/configuration.hh | 2 +- src/libutil/include/nix/util/error.hh | 2 +- .../include/nix/util/file-descriptor.hh | 1 + src/libutil/include/nix/util/serialise.hh | 1 + src/libutil/include/nix/util/sort.hh | 12 ++++++------ src/libutil/include/nix/util/topo-sort.hh | 2 +- src/libutil/include/nix/util/util.hh | 2 +- src/libutil/linux/linux-namespaces.cc | 4 ++-- src/libutil/logging.cc | 12 +++++++++--- src/libutil/unix/file-system.cc | 2 +- 37 files changed, 80 insertions(+), 64 deletions(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index daf4c7f7c4b8..b0cc92429c4e 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -51,11 +51,6 @@ Checks: - -bugprone-macro-parentheses # 1 warning - increment/decrement in conditions - -bugprone-inc-dec-in-conditions - # 2 warnings - std::move on forwarding reference (auto&&) in ranges lambdas - - -bugprone-move-forwarding-reference - # 2 warnings - coroutine pattern: co_await await(std::move(waitees)) then reuse. - # Relies on moved-from containers being empty (holds for libstdc++/libc++). - - -bugprone-use-after-move # 2 warnings - sorts Value* by ->string_view(), not by pointer value (false positive) - -bugprone-nondeterministic-pointer-iteration-order # 9 warnings - intentional std::bit_cast/memcpy on Value* arrays (evaluator hot path) @@ -68,7 +63,7 @@ Checks: # template; fires when T=unsigned char but that instantiation is correct. - -bugprone-unintended-char-ostream-output # - # Non-bugprone checks (also disabled to pass on current codebase): + # Non-bugprone checks (some disabled to pass on current codebase): # # 4 warnings - exceptions not derived from std::exception # All thrown exceptions must derive from std::exception @@ -77,6 +72,14 @@ Checks: # - cppcoreguidelines-pro-type-cstyle-cast # 11 warnings - coroutine lambdas with captures (intentional pattern in async goal/store code) # - cppcoreguidelines-avoid-capturing-lambda-coroutines + - performance-noexcept-swap + - performance-noexcept-move-constructor + - performance-noexcept-destructor + - performance-use-std-move + - misc-throw-by-value-catch-by-reference + - cppcoreguidelines-missing-std-forward + - android-cloexec-open + - android-cloexec-pipe2 # Custom nix checks (when added) - nix-* @@ -93,3 +96,5 @@ CheckOptions: bugprone-unsafe-functions.CustomFunctions: > ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; + +ExtraArgs: ["-Werror=unnecessary-virtual-specifier"] diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index c3a3f2dd53b1..8362e6850892 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -20,8 +20,8 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) auto delTmpDir = std::make_unique(tmpDir, true); auto nixpkgs = tmpDir / "pkgs"; auto nixos = tmpDir / "cfg"; - std::filesystem::create_directories(nixpkgs); - std::filesystem::create_directories(nixos); + nix::createDirs(nixpkgs); + nix::createDirs(nixos); std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string(); std::string nixosEntry = "nixos-config=" + nixos.string(); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a419530c6dd8..63f0c148870c 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -106,7 +106,7 @@ struct AttrDb } template - AttrId doSQLite(F && fun) + AttrId doSQLite(const F & fun) { if (failed) return 0; diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 67e8d5ea1ded..22c622526c46 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -97,7 +97,7 @@ void EvalErrorBuilder::debugThrow() auto error = std::move(this->error); delete this; - throw error; + throw std::move(error); } template diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 700d9a1ebe6b..4cd90a2a5f84 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3465,7 +3465,7 @@ void forceNoNullByte(std::string_view s, std::function pos) if (pos) { error.atPos(pos()); } - throw error; + throw std::move(error); } } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 693b1946ee46..1acc591b05b7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -162,14 +162,14 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) - throw errMsg; + throw std::move(errMsg); Outputs result; for (auto elem : outTI->listView()) { if (elem->type() != nString) - throw errMsg; + throw std::move(errMsg); auto out = outputs.find(elem->string_view()); if (out == outputs.end()) - throw errMsg; + throw std::move(errMsg); result.insert(*out); } return result; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 68c8f6543f59..4a360970ba21 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -44,7 +44,7 @@ NIX_DECLARE_CONFIG_SERIALISER(Diagnose) * @throws The error returned by mkError if level is `Fatal` and mkError returns a value */ template -void diagnose(const Setting & setting, F && mkError) +void diagnose(const Setting & setting, const F & mkError) { auto withError = [&](bool fatal, auto && handler) { auto maybeError = mkError(fatal); @@ -64,7 +64,7 @@ void diagnose(const Setting & setting, F && mkError) withError(false, [](auto && error) { logWarning(error.info()); }); return; case Diagnose::Fatal: - withError(true, [](auto && error) { throw std::move(error); }); + withError(true, [](auto && error) { throw std::forward(error); }); return; } } diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 07fbed403c2f..b13c00ad541e 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -557,7 +557,7 @@ public: std::numeric_limits::max()); if (pos) err.atPos(positions[pos]); - throw err; + throw std::move(err); } std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart); }; diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index f9bd06589e42..2482d53ea041 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -89,7 +89,7 @@ public: * @see https://github.com/NixOS/nix/issues/14642 */ template - void visit(F && f) + void visit(const F & f) { std::visit( overloaded{ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1ff6..5683d13d5c4b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -694,7 +694,7 @@ static RegisterPrimOp primop_isPath({ }); template -static inline void withExceptionContext(Trace trace, Callable && func) +static inline void withExceptionContext(Trace trace, const Callable & func) { try { func(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6d76cd9d275e..fc4c8a5bb507 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -92,6 +92,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) co_await await(std::move(waitees)); if (nrFailed == 0) { + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); co_await await(std::move(waitees)); @@ -111,6 +113,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } } + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 7651282e3e4f..af245d90187c 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -58,13 +58,13 @@ Goal::ChildEvent Goal::ChildEvents::popChildEvent() using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; -Co::Co(Co && rhs) +Co::Co(Co && rhs) noexcept { this->handle = rhs.handle; rhs.handle = nullptr; } -Co & Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) noexcept { if (handle) { handle.promise().alive = false; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 4cb42975fe29..90273493e288 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,7 +146,7 @@ Goal::Co PathSubstitutionGoal::init() } if (lastStoresException.has_value()) { if (!worker.settings.tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 002bb1a30d79..3fe196d08005 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -61,7 +61,7 @@ std::shared_ptr Worker::initGoalIfNeeded(std::weak_ptr & goal_weak, Args & if (auto goal = goal_weak.lock()) return goal; - auto goal = std::make_shared(args...); + auto goal = std::make_shared(std::forward(args)...); goal_weak = goal; wakeUp(goal); return goal; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ff5a89f135bb..713de4969a94 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -214,7 +214,7 @@ void HttpBinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToHTTP err(e.message()); err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 6ddc73250d34..3e5a3139d5ff 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -255,8 +255,8 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - Co & operator=(Co &&); - Co(Co && rhs); + Co & operator=(Co &&) noexcept; + Co(Co && rhs) noexcept; Co & operator=(const Co &) = delete; Co(const Co & rhs) = delete; ~Co(); diff --git a/src/libstore/include/nix/store/sqlite.hh b/src/libstore/include/nix/store/sqlite.hh index 789e82174627..0a2d21b12849 100644 --- a/src/libstore/include/nix/store/sqlite.hh +++ b/src/libstore/include/nix/store/sqlite.hh @@ -207,7 +207,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); * database is busy. */ template -T retrySQLite(F && fun) +T retrySQLite(const F & fun) { time_t nextWarning = time(nullptr) + 1; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 51708a2cbce4..8db977b6fce0 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -149,7 +149,7 @@ querySubstitutablePathInfosAsync(Store & store, const StorePathCAMap & paths, Su } if (lastStoresException.has_value()) { if (!settings.getWorkerSettings().tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 733157524176..f6d300c0936f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -173,7 +173,7 @@ void S3BinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToS3 err(e.message()); err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 3fd1f798ea74..c65fc240c66f 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -47,7 +47,7 @@ SQLiteError::SQLiteError( exp.err.msg = HintFmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" : "SQLite database '%s' is busy", path ? path : "(in-memory)"); - throw exp; + throw std::move(exp); } else throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3f367b19a16c..3c23156769b7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -55,21 +55,19 @@ StoreConfigBase::StoreDirSetting::StoreDirSetting(Config * options, FilePathType switch (pathType) { case FilePathType::Unix: - return canonStoreDir( - envOverrides.transform([](auto && s) { return os_string_to_string(std::move(s)); }) - .value_or(NIX_STORE_DIR)); + return canonStoreDir(envOverrides.transform([](const auto & s) { return os_string_to_string(s); }) + .value_or(NIX_STORE_DIR)); case FilePathType::Native: - return canonStoreDir( - envOverrides.transform([](auto && s) { return std::filesystem::path(std::move(s)); }) - .or_else([]() -> std::optional { + return canonStoreDir(envOverrides.transform([](const auto & s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { #ifdef _WIN32 - return windows::known_folders::getProgramData() / "nix" / "store"; + return windows::known_folders::getProgramData() / "nix" / "store"; #else - return std::filesystem::path{NIX_STORE_DIR}; + return std::filesystem::path{NIX_STORE_DIR}; #endif - }) - .value()); + }) + .value()); } assert(false); }(), diff --git a/src/libstore/unix/build/child.cc b/src/libstore/unix/build/child.cc index 3a704e6edf2c..f603b671e440 100644 --- a/src/libstore/unix/build/child.cc +++ b/src/libstore/unix/build/child.cc @@ -26,7 +26,7 @@ void commonChildInit() throw SysError("cannot dup stderr into stdout"); /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR | O_CLOEXEC); if (fdDevNull == -1) throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 7d348af31975..3e9ad5434754 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -974,7 +974,7 @@ void DerivationBuilderImpl::openSlave() { std::string slaveName = getPtsName(builderOut.get()); - AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY | O_CLOEXEC); if (!builderOut) throw SysError("opening pseudoterminal slave"); @@ -1056,7 +1056,7 @@ void DerivationBuilderImpl::processSandboxSetupMessages() FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); - throw ex; + throw std::move(ex); } debug("sandbox setup: " + msg); msgs.push_back(std::move(msg)); diff --git a/src/libutil-tests/file-descriptor.cc b/src/libutil-tests/file-descriptor.cc index 808bb31d8e99..c385d2960f01 100644 --- a/src/libutil-tests/file-descriptor.cc +++ b/src/libutil-tests/file-descriptor.cc @@ -130,7 +130,7 @@ TEST(ReadLine, TreatsEioAsEof) ASSERT_EQ(unlockpt(master), 0); // Open and immediately close the slave to trigger EIO on the master. - int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + int slave = open(ptsname(master), O_RDWR | O_NOCTTY | O_CLOEXEC); ASSERT_NE(slave, -1); close(slave); @@ -153,7 +153,7 @@ TEST(ReadLine, PartialLineBeforeEio) ASSERT_EQ(grantpt(master), 0); ASSERT_EQ(unlockpt(master), 0); - int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + int slave = open(ptsname(master), O_RDWR | O_NOCTTY | O_CLOEXEC); ASSERT_NE(slave, -1); // Write a partial line (no terminator) from the slave, then close it. diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index acd273ca79d9..eff35721b2a1 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -237,6 +237,7 @@ AutoCloseFD::AutoCloseFD(AutoCloseFD && that) noexcept that.fd = INVALID_DESCRIPTOR; } +// NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw AutoCloseFD & AutoCloseFD::operator=(AutoCloseFD && that) { close(); diff --git a/src/libutil/include/nix/util/async.hh b/src/libutil/include/nix/util/async.hh index 7abd446f7347..3f6be9b48cc7 100644 --- a/src/libutil/include/nix/util/async.hh +++ b/src/libutil/include/nix/util/async.hh @@ -88,7 +88,7 @@ asio::awaitable callbackToAwaitable(F && initiate) } template -asio::awaitable forEachAsync(Range && range, F && f) +asio::awaitable forEachAsync(Range && range, const F & f) { /* This code only runs on a strand - we don't do multithreaded executors, so no need for synchronisation. */ @@ -102,7 +102,7 @@ asio::awaitable forEachAsync(Range && range, F && f) co_await asio::async_initiate( [&](auto handler) { auto h = std::make_shared(std::move(handler)); - for (auto && elt : range) { + for (auto && elt : std::forward(range)) { asio::co_spawn(executor, f(elt), [executor, h, &err, &pending](std::exception_ptr ex) { if (ex && !err) err = ex; diff --git a/src/libutil/include/nix/util/closure.hh b/src/libutil/include/nix/util/closure.hh index 586acfff7e70..5078a02d0795 100644 --- a/src/libutil/include/nix/util/closure.hh +++ b/src/libutil/include/nix/util/closure.hh @@ -26,7 +26,7 @@ template using GetEdgesAsync = fun>(const T & elt)>; template -auto computeClosure(std::set startElts, std::set & res, GetEdgesAsync getEdges, CompletionToken && token) +auto computeClosure(std::set startElts, std::set & res, GetEdgesAsync getEdges, CompletionToken token) { auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) { auto executor = asio::make_strand(asio::get_associated_executor(handler)); diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 19d678601b1e..5dc98904a4d0 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -578,7 +578,7 @@ struct ExperimentalFeatureSettings : Config */ template requires std::invocable && std::convertible_to, std::string> - void require(const ExperimentalFeature & feature, GetReason && getReason) const + void require(const ExperimentalFeature & feature, const GetReason & getReason) const { if (isEnabled(feature)) return; diff --git a/src/libutil/include/nix/util/error.hh b/src/libutil/include/nix/util/error.hh index b01c18e58da4..2a846fed17cf 100644 --- a/src/libutil/include/nix/util/error.hh +++ b/src/libutil/include/nix/util/error.hh @@ -126,7 +126,7 @@ protected: public: BaseError(const BaseError &) = default; BaseError & operator=(const BaseError &) = default; - BaseError & operator=(BaseError &&) = default; + BaseError & operator=(BaseError &&) noexcept = default; template BaseError(unsigned int status, Args &&... args) diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index f92e737fcda5..a6796f22691e 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -262,6 +262,7 @@ public: AutoCloseFD(AutoCloseFD && fd) noexcept; ~AutoCloseFD(); AutoCloseFD & operator=(const AutoCloseFD & fd) = delete; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw because of close() AutoCloseFD & operator=(AutoCloseFD && fd); Descriptor get() const; explicit operator bool() const; diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 761a0fe6ed88..ed84c767e8b3 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -177,6 +177,7 @@ struct FdSink : BufferedSink FdSink(const FdSink &) = delete; FdSink & operator=(const FdSink &) = delete; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) - can throw FdSink & operator=(FdSink && s) { flush(); diff --git a/src/libutil/include/nix/util/sort.hh b/src/libutil/include/nix/util/sort.hh index 2a4eb6e7c98e..77a3ea59a50d 100644 --- a/src/libutil/include/nix/util/sort.hh +++ b/src/libutil/include/nix/util/sort.hh @@ -123,7 +123,7 @@ void insertionsort(Iter begin, Iter end, Comparator comp = {}) * to the specified comparator. */ template>> -Iter strictlyDecreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) +Iter strictlyDecreasingPrefix(Iter begin, Iter end, const Comparator & comp = {}) { if (begin == end) return begin; @@ -138,7 +138,7 @@ Iter strictlyDecreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter strictlyDecreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) +Iter strictlyDecreasingSuffix(Iter begin, Iter end, const Comparator & comp = {}) { if (begin == end) return end; @@ -153,9 +153,9 @@ Iter strictlyDecreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter weaklyIncreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) +Iter weaklyIncreasingPrefix(Iter begin, Iter end, const Comparator & comp = {}) { - return strictlyDecreasingPrefix(begin, end, std::not_fn(std::forward(comp))); + return strictlyDecreasingPrefix(begin, end, std::not_fn(comp)); } /** @@ -163,9 +163,9 @@ Iter weaklyIncreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter weaklyIncreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) +Iter weaklyIncreasingSuffix(Iter begin, Iter end, const Comparator & comp = {}) { - return strictlyDecreasingSuffix(begin, end, std::not_fn(std::forward(comp))); + return strictlyDecreasingSuffix(begin, end, std::not_fn(comp)); } /** diff --git a/src/libutil/include/nix/util/topo-sort.hh b/src/libutil/include/nix/util/topo-sort.hh index 6218b66a5023..12031e071535 100644 --- a/src/libutil/include/nix/util/topo-sort.hh +++ b/src/libutil/include/nix/util/topo-sort.hh @@ -20,7 +20,7 @@ using TopoSortResult = std::variant, Cycle>; template F> requires std::same_as>, std::set> -TopoSortResult topoSort(std::set items, F && getChildren) +TopoSortResult topoSort(std::set items, const F & getChildren) { std::vector sorted; decltype(items) visited, parents; diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 144c83cc0c9f..8d26ab1a1efd 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -31,7 +31,7 @@ template auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = {parts...}; + std::string_view views[sizeof...(parts)] = {std::forward(parts)...}; return concatStringsSep({}, views); } diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 9c96c5bcb768..26a7479050ad 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -95,11 +95,11 @@ void saveMountNamespace() { static std::once_flag done; std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY | O_CLOEXEC); if (!fdSavedMountNamespace) throw SysError("saving parent mount namespace"); - fdSavedRoot = open("/proc/self/root", O_RDONLY); + fdSavedRoot = open("/proc/self/root", O_RDONLY | O_CLOEXEC); }); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 58c78df34986..66038e25e27f 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -359,9 +359,15 @@ std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool } }; - AutoCloseFD fd = std::filesystem::is_socket(path) - ? connect(path) - : toDescriptor(open(path.string().c_str(), O_CREAT | O_APPEND | O_WRONLY, 0644)); + AutoCloseFD fd = std::filesystem::is_socket(path) ? connect(path) + : toDescriptor(open( + path.string().c_str(), + O_CREAT | O_APPEND | O_WRONLY +#ifndef _WIN32 + | O_CLOEXEC +#endif + , + 0644)); if (!fd) throw SysError("opening log file %1%", PathFmt(path)); diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 9181f4466e1b..f9417bb7e9d4 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -211,7 +211,7 @@ static void _deletePath( throw; } - int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) throw SysError("opening directory %1%", PathFmt(path)); AutoCloseDir dir(fdopendir(fd)); From 8b974a35bddd047b8893d432ed951790765c4377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 26 Apr 2026 20:03:10 +0200 Subject: [PATCH 066/364] tests/functional/json: fix script(1) invocation for util-linux 2.42 util-linux 2.42 (commit 7268e79b) added "+" to the getopt string of script(1), so option parsing stops at the first non-option argument. The previous `script -e -q /dev/null -c CMD` ordering therefore treats `-c CMD` as extra positional arguments and fails with "unexpected number of arguments". Place all options before the argument, which works on both old and new util-linux. --- tests/functional/json.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/json.sh b/tests/functional/json.sh index 49992e0d9324..157395e11373 100644 --- a/tests/functional/json.sh +++ b/tests/functional/json.sh @@ -51,7 +51,7 @@ if type script &>/dev/null; then if [[ $acceptsCommandFlag -eq 0 ]]; then script -e -q /dev/null "$@" else - script -e -q /dev/null -c "$(shellEscapeArray "$@")" + script -e -q -c "$(shellEscapeArray "$@")" /dev/null fi } runScript nix eval --json --expr "{ a.b.c = true; }" > "$TEST_HOME/actual.json" From 456764609d0c696ce78758514cbeabab9917459d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 26 Apr 2026 20:53:13 +0200 Subject: [PATCH 067/364] doc/rl-next: add release note for blocking *listxattr in sandbox The seccomp filter change is user-visible: builds on SELinux hosts may now behave differently (correctly) and tools probing xattr support via listxattr will see ENOTSUP. Document this so users can trace behavior changes back to this PR. --- doc/manual/rl-next/seccomp-block-listxattr.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/manual/rl-next/seccomp-block-listxattr.md diff --git a/doc/manual/rl-next/seccomp-block-listxattr.md b/doc/manual/rl-next/seccomp-block-listxattr.md new file mode 100644 index 000000000000..2455ed4f825f --- /dev/null +++ b/doc/manual/rl-next/seccomp-block-listxattr.md @@ -0,0 +1,10 @@ +--- +synopsis: "Linux sandbox: also block `listxattr` syscalls" +prs: [15743] +--- + +The Linux sandbox now also returns `ENOTSUP` for `listxattr`, +`llistxattr` and `flistxattr`, matching the existing treatment of +`getxattr`/`setxattr`/`removexattr`. This prevents host xattrs (e.g. +`security.selinux`) from leaking into builds and fixes tools such as +`mkfs.ubifs` that probe xattr support via `listxattr`. From 02df27e1cff20c140ea216d0f7ff50325c4a9c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 18:54:01 +0200 Subject: [PATCH 068/364] rust-installer: 2.34.5 -> 2.34.6 2.34.6 dropped build.rs/env!() embedding in favour of appending the closure via scripts/pack post-build, so the Rust compile is independent of the embedded Nix and stays cacheable across revisions. Adapt the packaging accordingly. --- packaging/rust-installer/default.nix | 98 +++++++++++++++++----------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix index dcfbf4badc31..98aec45e9e53 100644 --- a/packaging/rust-installer/default.nix +++ b/packaging/rust-installer/default.nix @@ -4,61 +4,85 @@ { lib, stdenv, + buildPackages, + runCommand, rustPlatform, fetchFromGitHub, tarball, }: let - installerVersion = "2.34.5"; + installerVersion = "2.34.6"; src = fetchFromGitHub { owner = "NixOS"; repo = "nix-installer"; tag = installerVersion; - hash = "sha256-+gM241qQOzQlOnP0a7d47z3iRf9+yNjbBJCLIWWNX+c="; + hash = "sha256-aTaz8EtHexvke7tGr5MfeKy9g7AraIAFN+dPApm+fds="; }; -in -rustPlatform.buildRustPackage { - pname = "nix-installer"; - version = tarball.passthru.nixVersion; + # Bare binary: no Nix closure yet. Appended below via `pack`, so the + # (expensive) Rust compile is independent of the embedded Nix and + # stays cacheable across Nix revisions. + bare = rustPlatform.buildRustPackage { + pname = "nix-installer-bare"; + version = installerVersion; - inherit src; + inherit src; - cargoHash = "sha256-6pt2f7wznH672L5+SkbA5GA6Sxvk1KamAf3erGqZlLU="; + cargoHash = "sha256-/mNXkeZVuYsqd0TiUa7bzSP4xpKh0Fqga9EpasPbrzU="; - doCheck = false; + doCheck = false; - env = { - NIX_TARBALL_PATH = "${tarball}/nix.tar.zst"; - NIX_STORE_PATH = tarball.passthru.nixStorePath; - NSS_CACERT_STORE_PATH = tarball.passthru.cacertStorePath; - NIX_VERSION = tarball.passthru.nixVersion; - } - // lib.optionalAttrs stdenv.hostPlatform.isDarwin { - # Drop the unused libiconv dylib the darwin stdenv injects; the - # binary must run before `/nix/store` exists. - NIX_LDFLAGS = "-dead_strip_dylibs"; + env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + ''; }; +in - postInstall = '' - install -m755 nix-installer.sh $out/bin/nix-installer.sh +runCommand "nix-installer-${tarball.passthru.nixVersion}" + { + nativeBuildInputs = [ + buildPackages.python3 + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + buildPackages.darwin.sigtool + buildPackages.darwin.cctools + ]; - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products - echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products - ''; + # The appended payload contains store-path strings on purpose; don't + # let the reference scanner pull the whole Nix closure into this + # derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; - # The binary embeds store-path strings (`NIX_STORE_PATH`, …) on - # purpose; don't let the reference scanner pull the whole Nix - # closure into this derivation's runtime closure. - __structuredAttrs = true; - unsafeDiscardReferences.out = true; + passthru = { inherit bare; }; - meta = { - description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; - homepage = "https://github.com/NixOS/nix-installer"; - license = lib.licenses.lgpl21Only; - mainProgram = "nix-installer"; - }; -} + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; + } + '' + mkdir -p $out/bin $out/nix-support + + python3 ${src}/scripts/pack \ + --input ${bare}/bin/nix-installer \ + --tarball ${tarball}/nix.tar.zst \ + --nix-store-path ${tarball.passthru.nixStorePath} \ + --cacert-store-path ${tarball.passthru.cacertStorePath} \ + --nix-version ${tarball.passthru.nixVersion} \ + --output $out/bin/nix-installer + + install -m755 ${bare}/bin/nix-installer.sh $out/bin/nix-installer.sh + + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + '' From 498f96d1457de6d96b7e2b6aeeaee64c34c3a1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 21:35:04 +0200 Subject: [PATCH 069/364] libutil: Use poll() in FdSource::hasData() to avoid fd_set overflow FD_SET writes past the stack fd_set when fd >= FD_SETSIZE. hasData() runs before every frame in withFramedSink(), so clients with many open fds would corrupt the stack (or abort under glibc _FORTIFY_SOURCE) during addToStore(). poll() has no such limit; Windows keeps select() since its fd_set is a bounded handle array. --- src/libutil/serialise.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6c77c15fe584..4ff2d63f2f86 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -224,6 +224,9 @@ bool FdSource::hasData() return true; while (true) { +#ifdef _WIN32 + /* Windows' fd_set is a bounded handle array, so FD_SET can't + overflow; on Unix use poll() since fd may exceed FD_SETSIZE. */ fd_set fds; FD_ZERO(&fds); Socket sock = toSocket(fd); @@ -240,6 +243,20 @@ bool FdSource::hasData() throw SysError("polling file descriptor"); } return FD_ISSET(sock, &fds); +#else + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + auto n = poll(&pfd, 1, 0); + if (n < 0) { + if (errno == EINTR) + continue; + throw SysError("polling file descriptor"); + } + return n > 0 && (pfd.revents & (POLLIN | POLLHUP | POLLERR)) != 0; +#endif } } From 934a7af2877ac60207453bdf353092cf267a6ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 23:27:08 +0200 Subject: [PATCH 070/364] dependabot: enable Nix ecosystem for flake.lock updates Dependabot recently gained support for updating Nix flake inputs. Enable it so we get automated weekly bumps of flake.lock instead of having to update inputs manually. Group all flake inputs into a single pull request to avoid a flood of one-PR-per-input updates and to keep CI load and review effort low. --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a1f2..1880a89da556 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "nix" + directory: "/" + schedule: + interval: "weekly" + groups: + flake-inputs: + patterns: + - "*" From f2720c2037a2fdb241699c855d8d1cd387032bf3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:54:30 +0000 Subject: [PATCH 071/364] build(deps): bump cachix/install-nix-action from 31.10.4 to 31.10.5 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.4 to 31.10.5. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/616559265b40713947b9c190a8ff4b507b5df49b...ab739621df7a23f52766f9ccc97f38da6b7af14f) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07329d16a202..49fff49b9737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: - name: Looking up the installer tarball URL id: installer-tarball-url run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -255,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From 741de5d7481207eb2d761c17c3642711235d589e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:55:24 +0000 Subject: [PATCH 072/364] build(deps): bump the flake-inputs group with 2 updates Bumps the flake-inputs group with 2 updates: [flake-parts](https://github.com/hercules-ci/flake-parts) and [git-hooks-nix](https://github.com/cachix/git-hooks.nix). Updates `flake-parts` from `205b12d` to `3107b77` - [Commits](https://github.com/hercules-ci/flake-parts/compare/205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9...3107b77cd68437b9a76194f0f7f9c55f2329ca5b) Updates `git-hooks-nix` from `aa9f40c` to `3cfd774` - [Commits](https://github.com/cachix/git-hooks.nix/compare/aa9f40c906904ebd83da78e7f328cd8aeaeae785...3cfd774b0a530725a077e17354fbdb87ea1c4aad) --- updated-dependencies: - dependency-name: flake-parts dependency-version: 3107b77cd68437b9a76194f0f7f9c55f2329ca5b dependency-type: direct:production dependency-group: flake-inputs - dependency-name: git-hooks-nix dependency-version: 3cfd774b0a530725a077e17354fbdb87ea1c4aad dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] --- flake.lock | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 4c0bf91927a4..1212049c3bea 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "lastModified": 1775087534, + "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", "type": "github" }, "original": { @@ -42,17 +42,14 @@ "gitignore": [], "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", + "lastModified": 1776796298, + "narHash": "sha256-PcRvlWayisPSjd0UcRQbhG8Oqw78AcPE6x872cPRHN8=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", + "rev": "3cfd774b0a530725a077e17354fbdb87ea1c4aad", "type": "github" }, "original": { From f63f6037c921af7da9cf4eaa1bb12f9a30981251 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:45:23 -0400 Subject: [PATCH 073/364] Misc gc/deletion improvements / fixes - Improve deletion logging - Fix informational messages from being duplicated - Add some debugging messages useful for diagnosing why a path wasn't deleted - Use `.contains` instead of `.count` - Add --skip-live alias to match Lix (be forgiving to people used to Lix flags and support --skip-live as an alias for --skip-alive. - Exclude outputs and drvs from deletion if passed a StorePathSet. - gcDeleteSpecific is no longer an indicator of operating on specific paths, rather the presence of a StorePathSet in pathsToDelete is. - Clarify gc comments around keep-derivations and keep-outputs Signed-off-by: Lisanna Dettwyler --- src/libstore/gc.cc | 56 ++++++++++++++++++++++++----------------- src/nix/store-delete.cc | 1 + 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 835d06864671..701fc66e69d0 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -385,8 +385,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Using `--ignore-liveness' with `--delete' can have unintended consequences if `keep-outputs' or `keep-derivations' are true (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + or derivers, respectively, even if they aren't in the + pathsToDelete). So disable them. */ + if (std::holds_alternative(options.pathsToDelete) && options.ignoreLiveness) { keepOutputs = false; keepDerivations = false; } @@ -608,14 +609,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Bail out if we've previously discovered that this path is alive. */ - if (alive.count(*path)) { + if (alive.contains(*path)) { + debug("cannot delete '%s' because '%s' is alive", printStorePath(start), printStorePath(*path)); alive.insert(start); return; } /* If we've previously deleted this path, we don't have to handle it again. */ - if (dead.count(*path)) + if (dead.contains(*path)) continue; auto markAlive = [&]() { @@ -636,19 +638,24 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) }; /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (roots.contains(*path)) { debug("cannot delete '%s' because it's a root", printStorePath(*path)); return markAlive(); } if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) + && !std::get(options.pathsToDelete).contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); return; + } { auto hashPart = path->hashPart(); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { + if (shared->tempRoots.contains(hashPart)) { debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); return markAlive(); } @@ -668,8 +675,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a - derivation, then visit the derivation outputs. */ + /* If keep-derivations is set and this is a derivation, then we only want to delete this derivation if + * we can also delete all its outputs, so visit the derivation outputs. */ if (keepDerivations && path->isDerivation()) { for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) if (maybeOutPath && isValidPath(*maybeOutPath) @@ -677,7 +684,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) enqueue(*maybeOutPath); } - /* If keep-outputs is set, then visit the derivers. */ + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ if (keepOutputs) { auto derivers = queryValidDerivers(*path); for (auto & i : derivers) @@ -707,27 +715,29 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::visit( overloaded{ [&](const StorePathSet & paths) { - for (auto & i : paths) { - switch (options.action) { - case GCOptions::gcDeleteDead: - printInfo("deleting garbage within specified paths..."); - break; - case GCOptions::gcDeleteSpecific: - printInfo("deleting specified paths..."); - break; - case GCOptions::gcReturnDead: - case GCOptions::gcReturnLive: - printInfo("determining live/dead paths..."); - } + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } + for (auto & i : paths) { maybeDeleteReferrersClosure(i); - if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) + if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " "nix-store --query --roots and nix-store --query --referrers", printStorePath(i)); + else if (!dead.contains(i)) + debug("cannot delete '%s' because it's still alive", printStorePath(i)); } }, [&](const GCOptions::WholeStore & _) { diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5eef2d1ad235..a1a387898491 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -20,6 +20,7 @@ struct CmdStoreDelete : StorePathsCommand addFlag({ .longName = "skip-alive", + .aliases = {"skip-live"}, .description = "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", .handler = {&options.action, GCOptions::gcDeleteDead}, From 87032c6dc4b539a0fc9bb50964965bc6c3554532 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 28 Apr 2026 23:52:19 +0300 Subject: [PATCH 074/364] libstore: Drop queryMissing from Worker::run This is no longer needed, since PathSubstitutionGoal now queries ValidPathInfo asynchronously and we can start the builds right away without querying the whole closure at once. This can significantly speed up the build startup. This was added in back in bbdf08bc0facb5157a10c794712dae7e5902be03 when pathinfo queries were done sequentially and in a blocking manner - this is not the case anymore. The asynchronous queries don't wait for a build slot to start, so the concurrency there is not limited and all path queries can complete as the worker loop naturally. This also acts as natural rate limiting to avoid allocating too many coroutines up-front and blowing up the memory usage (which also slows down the whole build to a crawl because fork() starts taking increasingly longer to complete due large anonymous memory mappings). --- src/libstore/build/worker.cc | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 3fe196d08005..208bf5ca3ebb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -295,28 +295,11 @@ void Worker::waitForCompletion(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; - - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back( - DerivedPath::Built{ - .drvPath = goal->drvReq, - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - store.queryMissing(topPaths); - debug("entered goal loop"); + for (std::shared_ptr goal : _topGoals) + topGoals.insert(std::move(goal)); while (1) { - checkInterrupt(); // TODO GC interface? From 8c03190cea1b479246514006a55eae4bdfd844c3 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Sun, 26 Apr 2026 18:47:53 +0200 Subject: [PATCH 075/364] include in src/libfetchers/fetchers.cc `sleep_for` is defined in --- src/libfetchers/fetchers.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 4eedafa22a24..310373a19adf 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -11,6 +11,7 @@ #include "nix/store/pathlocks.hh" #include "nix/util/environment-variables.hh" +#include #include namespace nix::fetchers { From 53a90079c1293fc07293b7f2e5df2c69c5df79a3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Apr 2026 20:52:06 +0300 Subject: [PATCH 076/364] libstore: Anchor all Store and StoreConfig vtables Classes with virtual functions defined in headers must have at least one out-of-line definition of a virtual function (also called "key function" [3]) to prevent having the vtable from having weak (vague [2]) linkage and causing a bunch of headaches on Darwin. For now this just limits to fixes for what would be required for nix-heuristic-gc, which compiles with pybind11 which uses hidden visibility for the dylib. In the future we'd certainly want to enable -Werror=weak-vtables to prevent such footguns. The pattern used here is basically the same as what LLVM does [1]. [1]: https://github.com/llvm/llvm-project/blob/710c297289b751857bdc0a8677437d575b403922/clang/lib/ExtractAPI/API.cpp#L177-L200 [2]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vague [3]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vague-vtable --- src/libstore/binary-cache-store.cc | 4 ++++ src/libstore/common-ssh-store-config.cc | 2 ++ src/libstore/dummy-store.cc | 10 ++++++++++ src/libstore/http-binary-cache-store.cc | 4 ++++ .../include/nix/store/binary-cache-store.hh | 6 ++++++ .../include/nix/store/common-ssh-store-config.hh | 4 ++++ .../include/nix/store/dummy-store-impl.hh | 4 ++++ src/libstore/include/nix/store/dummy-store.hh | 4 ++++ src/libstore/include/nix/store/gc-store.hh | 4 ++++ .../include/nix/store/http-binary-cache-store.hh | 6 ++++++ .../include/nix/store/indirect-root-store.hh | 4 ++++ .../include/nix/store/legacy-ssh-store.hh | 8 ++++++++ .../nix/store/local-binary-cache-store.hh | 4 ++++ src/libstore/include/nix/store/local-fs-store.hh | 6 ++++++ .../include/nix/store/local-overlay-store.hh | 6 ++++++ src/libstore/include/nix/store/local-store.hh | 6 +++++- src/libstore/include/nix/store/log-store.hh | 4 ++++ src/libstore/include/nix/store/remote-store.hh | 8 ++++++++ src/libstore/include/nix/store/ssh-store.hh | 8 ++++++++ src/libstore/include/nix/store/store-api.hh | 10 ++++++++++ .../include/nix/store/uds-remote-store.hh | 8 ++++++++ src/libstore/indirect-root-store.cc | 2 ++ src/libstore/legacy-ssh-store.cc | 4 ++++ src/libstore/local-binary-cache-store.cc | 8 ++++++++ src/libstore/local-fs-store.cc | 4 ++++ src/libstore/local-overlay-store.cc | 4 ++++ src/libstore/local-store.cc | 8 ++++++++ src/libstore/log-store.cc | 2 ++ src/libstore/remote-store.cc | 4 ++++ src/libstore/restricted-store.cc | 6 ++++++ src/libstore/ssh-store.cc | 16 ++++++++++++++++ src/libstore/uds-remote-store.cc | 4 ++++ 32 files changed, 181 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 64fe33536bbd..8ceb8f2151af 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -22,6 +22,10 @@ namespace nix { +void BinaryCacheStoreConfig::anchor() {} + +void BinaryCacheStore::anchor() {} + BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index ee1d3bf8acde..db3677151416 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -9,6 +9,8 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authorit { } +void CommonSSHStoreConfig::anchor() {} + SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 052ec9b1283e..2ee093d85be8 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -10,6 +10,10 @@ namespace nix { +void DummyStoreConfig::anchor() {} + +void DummyStore::anchor() {} + std::string DummyStoreConfig::doc() { return @@ -126,6 +130,10 @@ bool DummyStoreConfig::getReadOnly() const struct DummyStoreImpl : DummyStore { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; /** @@ -378,6 +386,8 @@ struct DummyStoreImpl : DummyStore } }; +void DummyStoreImpl::anchor() {} + ref DummyStore::Config::openDummyStore() const { return make_ref(ref{shared_from_this()}); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 713de4969a94..8f0c22f99856 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -23,6 +23,10 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } +void HttpBinaryCacheStoreConfig::anchor() {} + +void HttpBinaryCacheStore::anchor() {} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) : StoreConfig(params, FilePathType::Unix) , BinaryCacheStoreConfig(params) diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 7871ad03c884..3e8cf886c897 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -16,6 +16,10 @@ class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: BinaryCacheStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { @@ -92,6 +96,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; private: + void anchor() override; + std::vector> signers; protected: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 1e90c94afcbe..b622e82e26bf 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -10,6 +10,10 @@ class SSHMaster; struct CommonSSHStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: CommonSSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 8fdeeb362515..bec77a6bee68 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -15,6 +15,10 @@ struct MemorySourceAccessor; */ struct DummyStore : virtual Store { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index f76fb3d5c2b0..c8a212c75603 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -12,6 +12,10 @@ struct DummyStore; struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { +private: + void anchor() override; + +public: DummyStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 5e23f2052472..015478e79fd3 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -106,6 +106,10 @@ struct GCResults */ struct GcStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Garbage collection"; /** diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 748daec646b9..12465261caef 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -14,6 +14,10 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: LegacySSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , CommonSSHStoreConfig(params) @@ -63,6 +67,10 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this struct LegacySSHStore : public virtual Store { +private: + void anchor() override; + +public: using Config = LegacySSHStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 69a4bac1c8a9..181b33e4bdf8 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -9,6 +9,10 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this> makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { @@ -87,6 +89,10 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ virtual GcStore, virtual LogStore { +private: + void anchor() override; + +public: using Config = LocalFSStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index dfb1fb184a55..10b04937c3aa 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -7,6 +7,10 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { +private: + void anchor() override; + +public: LocalOverlayStoreConfig(const StringMap & params) : LocalOverlayStoreConfig("", params) { @@ -119,6 +123,8 @@ struct LocalOverlayStore : virtual LocalStore LocalOverlayStore(ref); private: + void anchor() override; + /** * The store beneath us. * diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index bf3437e95771..e5ecaf8a59de 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -35,8 +35,9 @@ struct LocalSettings; struct LocalBuildStoreConfig : virtual LocalFSStoreConfig { - private: + void anchor() override; + /** Input for computing the build directory. See `getBuildDir()`. */ @@ -89,6 +90,7 @@ struct LocalStoreConfig : std::enable_shared_from_this, LocalStoreConfig(const std::filesystem::path & path, const Params & params); private: + void anchor() override; /** * An indirection so that we don't need to refer to global settings @@ -177,6 +179,8 @@ public: class LocalStore : public virtual IndirectRootStore, public virtual GcStore { + void anchor() override; + public: using Config = LocalStoreConfig; diff --git a/src/libstore/include/nix/store/log-store.hh b/src/libstore/include/nix/store/log-store.hh index 2d81d02b10cc..e0acd9a04f62 100644 --- a/src/libstore/include/nix/store/log-store.hh +++ b/src/libstore/include/nix/store/log-store.hh @@ -7,6 +7,10 @@ namespace nix { struct LogStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Build log storage and retrieval"; /** diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 57beb9135f7a..144b4b8e4355 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -23,6 +23,10 @@ class RemoteFSAccessor; struct RemoteStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: RemoteStoreConfig(const Params & params, FilePathType pathType) : StoreConfig(params, pathType) { @@ -44,6 +48,10 @@ struct RemoteStoreConfig : virtual StoreConfig */ struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore { +private: + void anchor() override; + +public: using Config = RemoteStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 4ab88ca74cb7..324e85eb60f4 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -12,6 +12,10 @@ struct SSHStoreConfig : std::enable_shared_from_this, virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: SSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , RemoteStoreConfig(params, FilePathType::Unix) @@ -43,6 +47,10 @@ struct SSHStoreConfig : std::enable_shared_from_this, struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig { +private: + void anchor() override; + +public: MountedSSHStoreConfig(StringMap params); MountedSSHStoreConfig(const ParsedURL::Authority & authority, StringMap params); diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 97251eaee9c2..bfd4ffce2181 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -229,6 +229,12 @@ public: */ struct StoreConfig : public StoreConfigBase, public StoreDirConfig { +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor() = 0; + +public: using Params = StoreReference::Params; StoreConfig(const Params & params, FilePathType pathType); @@ -380,6 +386,10 @@ struct StoreConfig : public StoreConfigBase, public StoreDirConfig */ class Store : public std::enable_shared_from_this, public StoreDirConfig { + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor() = 0; + public: using Config = StoreConfig; diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 43931c628af8..8359f8fdf45b 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -27,6 +27,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this virtual LocalFSStoreConfig, virtual RemoteStoreConfig { +private: + void anchor() override; + +public: UDSRemoteStoreConfig(const std::filesystem::path & path, const Params & params); UDSRemoteStoreConfig(const Params & params); @@ -57,6 +61,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore { +private: + void anchor() override; + +public: using Config = UDSRemoteStoreConfig; ref config; diff --git a/src/libstore/indirect-root-store.cc b/src/libstore/indirect-root-store.cc index 7384456e286d..b203afb63dcb 100644 --- a/src/libstore/indirect-root-store.cc +++ b/src/libstore/indirect-root-store.cc @@ -2,6 +2,8 @@ namespace nix { +void IndirectRootStore::anchor() {} + void IndirectRootStore::makeSymlink(const std::filesystem::path & link, const std::filesystem::path & target) { /* Create directories up to `gcRoot'. */ diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a00914c4493d..cbcc42dbed75 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -24,6 +24,8 @@ LegacySSHStoreConfig::LegacySSHStoreConfig(const ParsedURL::Authority & authorit { } +void LegacySSHStoreConfig::anchor() {} + std::string LegacySSHStoreConfig::doc() { return @@ -37,6 +39,8 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; +void LegacySSHStore::anchor() {} + LegacySSHStore::LegacySSHStore(ref config) : Store{*config} , config{config} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 60298efdbbfd..4d79f613e4a5 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -52,6 +52,10 @@ StoreReference LocalBinaryCacheStoreConfig::getReference() const struct LocalBinaryCacheStore : virtual BinaryCacheStore { +private: + void anchor() override; + +public: using Config = LocalBinaryCacheStoreConfig; ref config; @@ -139,6 +143,10 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } +void LocalBinaryCacheStoreConfig::anchor() {} + +void LocalBinaryCacheStore::anchor() {} + ref LocalBinaryCacheStoreConfig::openStore() const { auto store = make_ref( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 3fc724f5fe74..77525d5416da 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -5,6 +5,10 @@ namespace nix { +void LocalFSStoreConfig::anchor() {} + +void LocalFSStore::anchor() {} + LocalFSStoreConfig::LocalFSStoreConfig(const std::filesystem::path & rootDir, const Params & params) : StoreConfig(params, FilePathType::Native) /* Default `?root` from `rootDir` if non set diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 8d1a16f91281..c0fee83d7d18 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -10,6 +10,10 @@ namespace nix { +void LocalOverlayStoreConfig::anchor() {} + +void LocalOverlayStore::anchor() {} + std::string LocalOverlayStoreConfig::doc() { return diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 75f485c31f22..53d7456f94f6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,14 @@ namespace nix { +void LocalStoreConfig::anchor() {} + +void LocalBuildStoreConfig::anchor() {} + +void LocalStore::anchor() {} + +void GcStore::anchor() {} + LocalStoreConfig::LocalStoreConfig(const std::filesystem::path & path, const Params & params) : StoreConfig(params, FilePathType::Native) , LocalFSStoreConfig(path, params) diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc index fd03bb30ea02..23e6563991d4 100644 --- a/src/libstore/log-store.cc +++ b/src/libstore/log-store.cc @@ -2,6 +2,8 @@ namespace nix { +void LogStore::anchor() {} + std::optional LogStore::getBuildLog(const StorePath & path) { auto maybePath = getBuildDerivationPath(path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 88c3847bb3cf..a2fb3a10d251 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -30,6 +30,8 @@ namespace nix { +void RemoteStoreConfig::anchor() {} + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) : Store{config} @@ -59,6 +61,8 @@ RemoteStore::RemoteStore(const Config & config) { } +void RemoteStore::anchor() {} + ref RemoteStore::openConnectionWrapper() { if (failed) { diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 03c130da9653..eaeca59a2bf4 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -38,6 +38,10 @@ bool RestrictionContext::isAllowed(const DerivedPath & req) */ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore { +private: + void anchor() override; + +public: ref config; ref next; @@ -157,6 +161,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor } }; +void RestrictedStore::anchor() {} + ref makeRestrictedStore(ref config, ref next, RestrictionContext & context) { return make_ref(config, next, context); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 945271b1a929..4138be6e383a 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -17,6 +17,10 @@ SSHStoreConfig::SSHStoreConfig(const ParsedURL::Authority & authority, const Par { } +void SSHStoreConfig::anchor() {} + +void MountedSSHStoreConfig::anchor() {} + std::string SSHStoreConfig::doc() { return @@ -39,6 +43,10 @@ StoreReference SSHStoreConfig::getReference() const struct alignas(8) /* Work around ASAN failures on i686-linux. */ SSHStore : virtual RemoteStore { +private: + void anchor() override; + +public: using Config = SSHStoreConfig; ref config; @@ -87,6 +95,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ }; }; +void SSHStore::anchor() {} + MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) : StoreConfig(params, FilePathType::Native) , RemoteStoreConfig(params, FilePathType::Native) @@ -128,6 +138,10 @@ std::string MountedSSHStoreConfig::doc() */ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { +private: + void anchor() override; + +public: using Config = MountedSSHStoreConfig; MountedSSHStore(ref config) @@ -187,6 +201,8 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore } }; +void MountedSSHStore::anchor() {} + ref SSHStore::Config::openStore() const { return make_ref(ref{shared_from_this()}); diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index e0b2d68ec37c..c4137df7c6aa 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -26,6 +26,10 @@ std::filesystem::path getDaemonSocketPath(const Store::Config & config) .value_or(config.getStateDir() / "daemon-socket" / "socket"); } +void UDSRemoteStoreConfig::anchor() {} + +void UDSRemoteStore::anchor() {} + UDSRemoteStoreConfig::UDSRemoteStoreConfig(const std::filesystem::path & path, const StoreReference::Params & params) : Store::Config{params, FilePathType::Native} , LocalFSStore::Config{params} From 80416579477c30db201b6f9f0e0d19428eab839b Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Tue, 28 Apr 2026 19:36:45 +0200 Subject: [PATCH 077/364] Migrate C++ error trace tests into functional/lang tests. Migrated 138 language-level error trace tests from C++ unit tests to functional characterization tests in tests/functional/lang/. Some tests were commented out due to insufficiant MACROs, but they work well as functional lang tests. Some tests had duplicates, only one copy is kept. The remaning C++ tests (TraceBuilder and NestedThrows) were kept as is, while ~1200 lines were removed from src/libexpr-tests/error_traces.cc. --- src/libexpr-tests/error_traces.cc | 1266 ----------------- tests/functional/lang/eval-fail-add-1.err.exp | 10 + tests/functional/lang/eval-fail-add-1.nix | 1 + tests/functional/lang/eval-fail-add-2.err.exp | 10 + tests/functional/lang/eval-fail-add-2.nix | 1 + tests/functional/lang/eval-fail-all-1.err.exp | 10 + tests/functional/lang/eval-fail-all-1.nix | 1 + tests/functional/lang/eval-fail-all-2.err.exp | 10 + tests/functional/lang/eval-fail-all-2.nix | 1 + tests/functional/lang/eval-fail-all-3.err.exp | 10 + tests/functional/lang/eval-fail-all-3.nix | 1 + tests/functional/lang/eval-fail-any-1.err.exp | 10 + tests/functional/lang/eval-fail-any-1.nix | 1 + tests/functional/lang/eval-fail-any-2.err.exp | 10 + tests/functional/lang/eval-fail-any-2.nix | 1 + tests/functional/lang/eval-fail-any-3.err.exp | 10 + tests/functional/lang/eval-fail-any-3.nix | 1 + .../lang/eval-fail-attrNames-1.err.exp | 10 + .../functional/lang/eval-fail-attrNames-1.nix | 1 + .../lang/eval-fail-attrValues-1.err.exp | 10 + .../lang/eval-fail-attrValues-1.nix | 1 + .../lang/eval-fail-baseNameOf-1.err.exp | 10 + .../lang/eval-fail-baseNameOf-1.nix | 1 + .../lang/eval-fail-bitAnd-1.err.exp | 10 + tests/functional/lang/eval-fail-bitAnd-1.nix | 1 + .../lang/eval-fail-bitAnd-2.err.exp | 10 + tests/functional/lang/eval-fail-bitAnd-2.nix | 1 + .../functional/lang/eval-fail-bitOr-1.err.exp | 10 + tests/functional/lang/eval-fail-bitOr-1.nix | 1 + .../functional/lang/eval-fail-bitOr-2.err.exp | 10 + tests/functional/lang/eval-fail-bitOr-2.nix | 1 + .../lang/eval-fail-bitXor-1.err.exp | 10 + tests/functional/lang/eval-fail-bitXor-1.nix | 1 + .../lang/eval-fail-bitXor-2.err.exp | 10 + tests/functional/lang/eval-fail-bitXor-2.nix | 1 + .../lang/eval-fail-catAttrs-1.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-1.nix | 1 + .../lang/eval-fail-catAttrs-2.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-2.nix | 1 + .../lang/eval-fail-catAttrs-3.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-3.nix | 1 + .../lang/eval-fail-catAttrs-4.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-4.nix | 5 + .../functional/lang/eval-fail-ceil-1.err.exp | 10 + tests/functional/lang/eval-fail-ceil-1.nix | 1 + .../lang/eval-fail-compareVersions-1.err.exp | 10 + .../lang/eval-fail-compareVersions-1.nix | 1 + .../lang/eval-fail-compareVersions-2.err.exp | 10 + .../lang/eval-fail-compareVersions-2.nix | 1 + .../lang/eval-fail-concatLists-1.err.exp | 10 + .../lang/eval-fail-concatLists-1.nix | 1 + .../lang/eval-fail-concatLists-2.err.exp | 10 + .../lang/eval-fail-concatLists-2.nix | 1 + .../lang/eval-fail-concatLists-3.err.exp | 10 + .../lang/eval-fail-concatLists-3.nix | 4 + .../lang/eval-fail-concatMap-1.err.exp | 10 + .../functional/lang/eval-fail-concatMap-1.nix | 1 + .../lang/eval-fail-concatMap-2.err.exp | 10 + .../functional/lang/eval-fail-concatMap-2.nix | 1 + .../lang/eval-fail-concatMap-3.err.exp | 14 + .../functional/lang/eval-fail-concatMap-3.nix | 1 + .../lang/eval-fail-concatMap-4.err.exp | 14 + .../functional/lang/eval-fail-concatMap-4.nix | 4 + .../lang/eval-fail-concatStringsSep-1.err.exp | 10 + .../lang/eval-fail-concatStringsSep-1.nix | 1 + .../lang/eval-fail-concatStringsSep-2.err.exp | 10 + .../lang/eval-fail-concatStringsSep-2.nix | 1 + .../lang/eval-fail-concatStringsSep-3.err.exp | 10 + .../lang/eval-fail-concatStringsSep-3.nix | 5 + .../lang/eval-fail-derivationStrict-1.err.exp | 10 + .../lang/eval-fail-derivationStrict-1.nix | 1 + .../eval-fail-derivationStrict-10.err.exp | 18 + .../lang/eval-fail-derivationStrict-10.nix | 6 + .../eval-fail-derivationStrict-11.err.exp | 18 + .../lang/eval-fail-derivationStrict-11.nix | 6 + .../eval-fail-derivationStrict-12.err.exp | 18 + .../lang/eval-fail-derivationStrict-12.nix | 5 + .../eval-fail-derivationStrict-13.err.exp | 18 + .../lang/eval-fail-derivationStrict-13.nix | 6 + .../eval-fail-derivationStrict-14.err.exp | 18 + .../lang/eval-fail-derivationStrict-14.nix | 6 + .../eval-fail-derivationStrict-15.err.exp | 18 + .../lang/eval-fail-derivationStrict-15.nix | 9 + .../eval-fail-derivationStrict-16.err.exp | 18 + .../lang/eval-fail-derivationStrict-16.nix | 7 + .../eval-fail-derivationStrict-17.err.exp | 18 + .../lang/eval-fail-derivationStrict-17.nix | 7 + .../eval-fail-derivationStrict-19.err.exp | 18 + .../lang/eval-fail-derivationStrict-19.nix | 7 + .../lang/eval-fail-derivationStrict-2.err.exp | 10 + .../lang/eval-fail-derivationStrict-2.nix | 1 + .../eval-fail-derivationStrict-20.err.exp | 20 + .../lang/eval-fail-derivationStrict-20.nix | 7 + .../eval-fail-derivationStrict-21.err.exp | 20 + .../lang/eval-fail-derivationStrict-21.nix | 10 + .../eval-fail-derivationStrict-22.err.exp | 18 + .../lang/eval-fail-derivationStrict-22.nix | 7 + .../lang/eval-fail-derivationStrict-3.err.exp | 16 + .../lang/eval-fail-derivationStrict-3.nix | 1 + .../lang/eval-fail-derivationStrict-4.err.exp | 11 + .../lang/eval-fail-derivationStrict-4.nix | 1 + .../lang/eval-fail-derivationStrict-5.err.exp | 13 + .../lang/eval-fail-derivationStrict-5.nix | 5 + .../lang/eval-fail-derivationStrict-6.err.exp | 13 + .../lang/eval-fail-derivationStrict-6.nix | 5 + .../lang/eval-fail-derivationStrict-7.err.exp | 18 + .../lang/eval-fail-derivationStrict-7.nix | 5 + .../lang/eval-fail-derivationStrict-8.err.exp | 18 + .../lang/eval-fail-derivationStrict-8.nix | 5 + .../lang/eval-fail-derivationStrict-9.err.exp | 18 + .../lang/eval-fail-derivationStrict-9.nix | 5 + tests/functional/lang/eval-fail-div-1.err.exp | 10 + tests/functional/lang/eval-fail-div-1.nix | 1 + tests/functional/lang/eval-fail-div-2.err.exp | 10 + tests/functional/lang/eval-fail-div-2.nix | 1 + tests/functional/lang/eval-fail-div-3.err.exp | 8 + tests/functional/lang/eval-fail-div-3.nix | 1 + .../functional/lang/eval-fail-elem-1.err.exp | 10 + tests/functional/lang/eval-fail-elem-1.nix | 1 + .../lang/eval-fail-elemAt-1.err.exp | 10 + tests/functional/lang/eval-fail-elemAt-1.nix | 1 + .../lang/eval-fail-elemAt-2.err.exp | 8 + tests/functional/lang/eval-fail-elemAt-2.nix | 1 + .../lang/eval-fail-elemAt-3.err.exp | 8 + tests/functional/lang/eval-fail-elemAt-3.nix | 1 + .../lang/eval-fail-filter-1.err.exp | 10 + tests/functional/lang/eval-fail-filter-1.nix | 1 + .../lang/eval-fail-filter-2.err.exp | 10 + tests/functional/lang/eval-fail-filter-2.nix | 1 + .../lang/eval-fail-filter-3.err.exp | 10 + tests/functional/lang/eval-fail-filter-3.nix | 1 + .../lang/eval-fail-filterSource-1.err.exp | 10 + .../lang/eval-fail-filterSource-1.nix | 1 + .../lang/eval-fail-filterSource-2.err.exp | 10 + .../lang/eval-fail-filterSource-2.nix | 1 + .../lang/eval-fail-filterSource-3.err.exp | 10 + .../lang/eval-fail-filterSource-3.nix | 1 + .../lang/eval-fail-filterSource-4.err.exp | 10 + .../lang/eval-fail-filterSource-4.nix | 1 + .../lang/eval-fail-filterSource-5.err.exp | 12 + .../lang/eval-fail-filterSource-5.nix | 1 + .../functional/lang/eval-fail-floor-1.err.exp | 10 + tests/functional/lang/eval-fail-floor-1.nix | 1 + .../lang/eval-fail-foldlPrime-1.err.exp | 10 + .../lang/eval-fail-foldlPrime-1.nix | 1 + .../lang/eval-fail-foldlPrime-2.err.exp | 10 + .../lang/eval-fail-foldlPrime-2.nix | 1 + .../lang/eval-fail-foldlPrime-3.err.exp | 8 + .../lang/eval-fail-foldlPrime-3.nix | 1 + .../lang/eval-fail-foldlPrime-4.err.exp | 24 + .../lang/eval-fail-foldlPrime-4.nix | 1 + .../lang/eval-fail-functionArgs-1.err.exp | 8 + .../lang/eval-fail-functionArgs-1.nix | 1 + .../lang/eval-fail-genList-1.err.exp | 10 + tests/functional/lang/eval-fail-genList-1.nix | 1 + .../lang/eval-fail-genList-2.err.exp | 10 + tests/functional/lang/eval-fail-genList-2.nix | 1 + .../lang/eval-fail-genList-3.err.exp | 20 + tests/functional/lang/eval-fail-genList-3.nix | 1 + .../lang/eval-fail-genList-4.err.exp | 8 + tests/functional/lang/eval-fail-genList-4.nix | 1 + .../lang/eval-fail-getAttr-1.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-1.nix | 1 + .../lang/eval-fail-getAttr-2.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-2.nix | 1 + .../lang/eval-fail-getAttr-3.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-3.nix | 1 + .../lang/eval-fail-getEnv-1.err.exp | 10 + tests/functional/lang/eval-fail-getEnv-1.nix | 1 + .../lang/eval-fail-groupBy-1.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-1.nix | 1 + .../lang/eval-fail-groupBy-2.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-2.nix | 1 + .../lang/eval-fail-groupBy-3.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-3.nix | 5 + .../lang/eval-fail-hasAttr-1.err.exp | 10 + tests/functional/lang/eval-fail-hasAttr-1.nix | 1 + .../lang/eval-fail-hasAttr-2.err.exp | 10 + tests/functional/lang/eval-fail-hasAttr-2.nix | 1 + .../lang/eval-fail-hashString-1.err.exp | 10 + .../lang/eval-fail-hashString-1.nix | 1 + .../lang/eval-fail-hashString-2.err.exp | 9 + .../lang/eval-fail-hashString-2.nix | 1 + .../lang/eval-fail-hashString-3.err.exp | 10 + .../lang/eval-fail-hashString-3.nix | 1 + .../functional/lang/eval-fail-head-1.err.exp | 10 + tests/functional/lang/eval-fail-head-1.nix | 1 + .../functional/lang/eval-fail-head-2.err.exp | 8 + tests/functional/lang/eval-fail-head-2.nix | 1 + .../lang/eval-fail-intersectAttrs-1.err.exp | 10 + .../lang/eval-fail-intersectAttrs-1.nix | 1 + .../lang/eval-fail-intersectAttrs-2.err.exp | 10 + .../lang/eval-fail-intersectAttrs-2.nix | 1 + .../lang/eval-fail-length-1.err.exp | 10 + tests/functional/lang/eval-fail-length-1.nix | 1 + .../lang/eval-fail-length-2.err.exp | 10 + tests/functional/lang/eval-fail-length-2.nix | 1 + .../lang/eval-fail-lessThan-1.err.exp | 8 + .../functional/lang/eval-fail-lessThan-1.nix | 1 + .../lang/eval-fail-lessThan-2.err.exp | 8 + .../functional/lang/eval-fail-lessThan-2.nix | 1 + .../lang/eval-fail-lessThan-3.err.exp | 10 + .../functional/lang/eval-fail-lessThan-3.nix | 1 + .../lang/eval-fail-listToAttrs-1.err.exp | 10 + .../lang/eval-fail-listToAttrs-1.nix | 1 + .../lang/eval-fail-listToAttrs-2.err.exp | 10 + .../lang/eval-fail-listToAttrs-2.nix | 1 + .../lang/eval-fail-listToAttrs-3.err.exp | 10 + .../lang/eval-fail-listToAttrs-3.nix | 1 + .../lang/eval-fail-listToAttrs-4.err.exp | 18 + .../lang/eval-fail-listToAttrs-4.nix | 1 + .../lang/eval-fail-listToAttrs-5.err.exp | 10 + .../lang/eval-fail-listToAttrs-5.nix | 1 + tests/functional/lang/eval-fail-map-1.err.exp | 10 + tests/functional/lang/eval-fail-map-1.nix | 1 + tests/functional/lang/eval-fail-map-2.err.exp | 10 + tests/functional/lang/eval-fail-map-2.nix | 1 + .../lang/eval-fail-mapAttrs-1.err.exp | 10 + .../functional/lang/eval-fail-mapAttrs-1.nix | 1 + .../lang/eval-fail-mapAttrs-2.err.exp | 4 + .../functional/lang/eval-fail-mapAttrs-2.nix | 1 + .../lang/eval-fail-mapAttrs-3.err.exp | 4 + .../functional/lang/eval-fail-mapAttrs-3.nix | 1 + .../lang/eval-fail-mapAttrs-4.err.exp | 16 + .../functional/lang/eval-fail-mapAttrs-4.nix | 1 + .../functional/lang/eval-fail-match-1.err.exp | 10 + tests/functional/lang/eval-fail-match-1.nix | 1 + .../functional/lang/eval-fail-match-2.err.exp | 10 + tests/functional/lang/eval-fail-match-2.nix | 1 + .../functional/lang/eval-fail-match-3.err.exp | 8 + tests/functional/lang/eval-fail-match-3.nix | 1 + tests/functional/lang/eval-fail-mul-1.err.exp | 10 + tests/functional/lang/eval-fail-mul-1.nix | 1 + tests/functional/lang/eval-fail-mul-2.err.exp | 10 + tests/functional/lang/eval-fail-mul-2.nix | 1 + .../lang/eval-fail-parseDrvName-1.err.exp | 10 + .../lang/eval-fail-parseDrvName-1.nix | 1 + .../lang/eval-fail-partition-1.err.exp | 10 + .../functional/lang/eval-fail-partition-1.nix | 1 + .../lang/eval-fail-partition-2.err.exp | 10 + .../functional/lang/eval-fail-partition-2.nix | 1 + .../lang/eval-fail-partition-3.err.exp | 10 + .../functional/lang/eval-fail-partition-3.nix | 1 + .../lang/eval-fail-pathExists-1.err.exp | 10 + .../lang/eval-fail-pathExists-1.nix | 1 + .../lang/eval-fail-pathExists-2.err.exp | 10 + .../lang/eval-fail-pathExists-2.nix | 1 + .../lang/eval-fail-placeholder-1.err.exp | 10 + .../lang/eval-fail-placeholder-1.nix | 1 + .../lang/eval-fail-removeAttrs-1.err.exp | 10 + .../lang/eval-fail-removeAttrs-1.nix | 1 + .../lang/eval-fail-removeAttrs-2.err.exp | 10 + .../lang/eval-fail-removeAttrs-2.nix | 1 + .../lang/eval-fail-removeAttrs-3.err.exp | 10 + .../lang/eval-fail-removeAttrs-3.nix | 1 + .../lang/eval-fail-replaceStrings-1.err.exp | 10 + .../lang/eval-fail-replaceStrings-1.nix | 1 + .../lang/eval-fail-replaceStrings-2.err.exp | 10 + .../lang/eval-fail-replaceStrings-2.nix | 1 + .../lang/eval-fail-replaceStrings-3.err.exp | 8 + .../lang/eval-fail-replaceStrings-3.nix | 1 + .../lang/eval-fail-replaceStrings-4.err.exp | 10 + .../lang/eval-fail-replaceStrings-4.nix | 1 + .../lang/eval-fail-replaceStrings-5.err.exp | 10 + .../lang/eval-fail-replaceStrings-5.nix | 1 + .../lang/eval-fail-replaceStrings-6.err.exp | 10 + .../lang/eval-fail-replaceStrings-6.nix | 1 + .../functional/lang/eval-fail-sort-1.err.exp | 10 + tests/functional/lang/eval-fail-sort-1.nix | 1 + .../functional/lang/eval-fail-sort-2.err.exp | 10 + tests/functional/lang/eval-fail-sort-2.nix | 1 + .../functional/lang/eval-fail-sort-3.err.exp | 8 + tests/functional/lang/eval-fail-sort-3.nix | 4 + .../functional/lang/eval-fail-sort-4.err.exp | 10 + tests/functional/lang/eval-fail-sort-4.nix | 4 + .../functional/lang/eval-fail-sort-5.err.exp | 26 + tests/functional/lang/eval-fail-sort-5.nix | 4 + .../functional/lang/eval-fail-sort-6.err.exp | 26 + tests/functional/lang/eval-fail-sort-6.nix | 4 + .../functional/lang/eval-fail-split-1.err.exp | 10 + tests/functional/lang/eval-fail-split-1.nix | 1 + .../functional/lang/eval-fail-split-2.err.exp | 10 + tests/functional/lang/eval-fail-split-2.nix | 1 + .../functional/lang/eval-fail-split-3.err.exp | 8 + tests/functional/lang/eval-fail-split-3.nix | 1 + .../lang/eval-fail-splitVersion-1.err.exp | 10 + .../lang/eval-fail-splitVersion-1.nix | 1 + .../lang/eval-fail-storePath-1.err.exp | 10 + .../functional/lang/eval-fail-storePath-1.nix | 1 + .../lang/eval-fail-stringLength-1.err.exp | 10 + .../lang/eval-fail-stringLength-1.nix | 1 + tests/functional/lang/eval-fail-sub-1.err.exp | 10 + tests/functional/lang/eval-fail-sub-1.nix | 1 + tests/functional/lang/eval-fail-sub-2.err.exp | 10 + tests/functional/lang/eval-fail-sub-2.nix | 1 + .../lang/eval-fail-substring-1.err.exp | 10 + .../functional/lang/eval-fail-substring-1.nix | 1 + .../lang/eval-fail-substring-2.err.exp | 10 + .../functional/lang/eval-fail-substring-2.nix | 1 + .../lang/eval-fail-substring-3.err.exp | 10 + .../functional/lang/eval-fail-substring-3.nix | 1 + .../lang/eval-fail-substring-4.err.exp | 8 + .../functional/lang/eval-fail-substring-4.nix | 1 + .../functional/lang/eval-fail-tail-1.err.exp | 10 + tests/functional/lang/eval-fail-tail-1.nix | 1 + .../functional/lang/eval-fail-tail-2.err.exp | 8 + tests/functional/lang/eval-fail-tail-2.nix | 1 + .../lang/eval-fail-toPath-1.err.exp | 10 + tests/functional/lang/eval-fail-toPath-1.nix | 1 + .../lang/eval-fail-toPath-2.err.exp | 10 + tests/functional/lang/eval-fail-toPath-2.nix | 1 + .../lang/eval-fail-toString-1.err.exp | 10 + .../functional/lang/eval-fail-toString-1.nix | 1 + .../lang/eval-fail-zipAttrsWith-1.err.exp | 10 + .../lang/eval-fail-zipAttrsWith-1.nix | 1 + .../lang/eval-fail-zipAttrsWith-2.err.exp | 10 + .../lang/eval-fail-zipAttrsWith-2.nix | 1 + .../lang/eval-fail-zipAttrsWith-3.err.exp | 8 + .../lang/eval-fail-zipAttrsWith-3.nix | 1 + .../lang/eval-fail-zipAttrsWith-4.err.exp | 22 + .../lang/eval-fail-zipAttrsWith-4.nix | 4 + 321 files changed, 2068 insertions(+), 1266 deletions(-) create mode 100644 tests/functional/lang/eval-fail-add-1.err.exp create mode 100644 tests/functional/lang/eval-fail-add-1.nix create mode 100644 tests/functional/lang/eval-fail-add-2.err.exp create mode 100644 tests/functional/lang/eval-fail-add-2.nix create mode 100644 tests/functional/lang/eval-fail-all-1.err.exp create mode 100644 tests/functional/lang/eval-fail-all-1.nix create mode 100644 tests/functional/lang/eval-fail-all-2.err.exp create mode 100644 tests/functional/lang/eval-fail-all-2.nix create mode 100644 tests/functional/lang/eval-fail-all-3.err.exp create mode 100644 tests/functional/lang/eval-fail-all-3.nix create mode 100644 tests/functional/lang/eval-fail-any-1.err.exp create mode 100644 tests/functional/lang/eval-fail-any-1.nix create mode 100644 tests/functional/lang/eval-fail-any-2.err.exp create mode 100644 tests/functional/lang/eval-fail-any-2.nix create mode 100644 tests/functional/lang/eval-fail-any-3.err.exp create mode 100644 tests/functional/lang/eval-fail-any-3.nix create mode 100644 tests/functional/lang/eval-fail-attrNames-1.err.exp create mode 100644 tests/functional/lang/eval-fail-attrNames-1.nix create mode 100644 tests/functional/lang/eval-fail-attrValues-1.err.exp create mode 100644 tests/functional/lang/eval-fail-attrValues-1.nix create mode 100644 tests/functional/lang/eval-fail-baseNameOf-1.err.exp create mode 100644 tests/functional/lang/eval-fail-baseNameOf-1.nix create mode 100644 tests/functional/lang/eval-fail-bitAnd-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitAnd-1.nix create mode 100644 tests/functional/lang/eval-fail-bitAnd-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitAnd-2.nix create mode 100644 tests/functional/lang/eval-fail-bitOr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitOr-1.nix create mode 100644 tests/functional/lang/eval-fail-bitOr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitOr-2.nix create mode 100644 tests/functional/lang/eval-fail-bitXor-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitXor-1.nix create mode 100644 tests/functional/lang/eval-fail-bitXor-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitXor-2.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-ceil-1.err.exp create mode 100644 tests/functional/lang/eval-fail-ceil-1.nix create mode 100644 tests/functional/lang/eval-fail-compareVersions-1.err.exp create mode 100644 tests/functional/lang/eval-fail-compareVersions-1.nix create mode 100644 tests/functional/lang/eval-fail-compareVersions-2.err.exp create mode 100644 tests/functional/lang/eval-fail-compareVersions-2.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-1.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-2.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-3.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-1.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-2.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-3.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-4.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-4.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-1.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-2.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-3.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-1.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-1.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-10.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-10.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-11.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-11.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-12.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-12.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-13.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-13.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-14.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-14.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-15.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-15.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-16.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-16.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-17.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-17.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-19.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-19.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-2.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-2.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-20.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-20.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-21.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-21.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-22.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-22.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-3.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-3.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-4.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-4.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-5.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-5.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-6.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-6.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-7.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-7.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-8.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-8.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-9.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-9.nix create mode 100644 tests/functional/lang/eval-fail-div-1.err.exp create mode 100644 tests/functional/lang/eval-fail-div-1.nix create mode 100644 tests/functional/lang/eval-fail-div-2.err.exp create mode 100644 tests/functional/lang/eval-fail-div-2.nix create mode 100644 tests/functional/lang/eval-fail-div-3.err.exp create mode 100644 tests/functional/lang/eval-fail-div-3.nix create mode 100644 tests/functional/lang/eval-fail-elem-1.err.exp create mode 100644 tests/functional/lang/eval-fail-elem-1.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-1.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-1.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-2.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-2.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-3.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-3.nix create mode 100644 tests/functional/lang/eval-fail-filter-1.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-1.nix create mode 100644 tests/functional/lang/eval-fail-filter-2.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-2.nix create mode 100644 tests/functional/lang/eval-fail-filter-3.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-3.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-1.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-1.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-2.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-2.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-3.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-3.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-4.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-4.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-5.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-5.nix create mode 100644 tests/functional/lang/eval-fail-floor-1.err.exp create mode 100644 tests/functional/lang/eval-fail-floor-1.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-1.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-1.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-2.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-2.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-3.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-3.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-4.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-4.nix create mode 100644 tests/functional/lang/eval-fail-functionArgs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-functionArgs-1.nix create mode 100644 tests/functional/lang/eval-fail-genList-1.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-1.nix create mode 100644 tests/functional/lang/eval-fail-genList-2.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-2.nix create mode 100644 tests/functional/lang/eval-fail-genList-3.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-3.nix create mode 100644 tests/functional/lang/eval-fail-genList-4.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-4.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-1.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-2.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-3.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-3.nix create mode 100644 tests/functional/lang/eval-fail-getEnv-1.err.exp create mode 100644 tests/functional/lang/eval-fail-getEnv-1.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-1.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-1.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-2.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-2.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-3.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-3.nix create mode 100644 tests/functional/lang/eval-fail-hasAttr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-hasAttr-1.nix create mode 100644 tests/functional/lang/eval-fail-hasAttr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-hasAttr-2.nix create mode 100644 tests/functional/lang/eval-fail-hashString-1.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-1.nix create mode 100644 tests/functional/lang/eval-fail-hashString-2.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-2.nix create mode 100644 tests/functional/lang/eval-fail-hashString-3.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-3.nix create mode 100644 tests/functional/lang/eval-fail-head-1.err.exp create mode 100644 tests/functional/lang/eval-fail-head-1.nix create mode 100644 tests/functional/lang/eval-fail-head-2.err.exp create mode 100644 tests/functional/lang/eval-fail-head-2.nix create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-length-1.err.exp create mode 100644 tests/functional/lang/eval-fail-length-1.nix create mode 100644 tests/functional/lang/eval-fail-length-2.err.exp create mode 100644 tests/functional/lang/eval-fail-length-2.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-1.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-1.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-2.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-2.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-3.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-3.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-5.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-5.nix create mode 100644 tests/functional/lang/eval-fail-map-1.err.exp create mode 100644 tests/functional/lang/eval-fail-map-1.nix create mode 100644 tests/functional/lang/eval-fail-map-2.err.exp create mode 100644 tests/functional/lang/eval-fail-map-2.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-match-1.err.exp create mode 100644 tests/functional/lang/eval-fail-match-1.nix create mode 100644 tests/functional/lang/eval-fail-match-2.err.exp create mode 100644 tests/functional/lang/eval-fail-match-2.nix create mode 100644 tests/functional/lang/eval-fail-match-3.err.exp create mode 100644 tests/functional/lang/eval-fail-match-3.nix create mode 100644 tests/functional/lang/eval-fail-mul-1.err.exp create mode 100644 tests/functional/lang/eval-fail-mul-1.nix create mode 100644 tests/functional/lang/eval-fail-mul-2.err.exp create mode 100644 tests/functional/lang/eval-fail-mul-2.nix create mode 100644 tests/functional/lang/eval-fail-parseDrvName-1.err.exp create mode 100644 tests/functional/lang/eval-fail-parseDrvName-1.nix create mode 100644 tests/functional/lang/eval-fail-partition-1.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-1.nix create mode 100644 tests/functional/lang/eval-fail-partition-2.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-2.nix create mode 100644 tests/functional/lang/eval-fail-partition-3.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-3.nix create mode 100644 tests/functional/lang/eval-fail-pathExists-1.err.exp create mode 100644 tests/functional/lang/eval-fail-pathExists-1.nix create mode 100644 tests/functional/lang/eval-fail-pathExists-2.err.exp create mode 100644 tests/functional/lang/eval-fail-pathExists-2.nix create mode 100644 tests/functional/lang/eval-fail-placeholder-1.err.exp create mode 100644 tests/functional/lang/eval-fail-placeholder-1.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-1.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-1.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-2.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-2.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-3.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-3.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-4.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-4.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-5.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-5.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-6.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-6.nix create mode 100644 tests/functional/lang/eval-fail-sort-1.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-1.nix create mode 100644 tests/functional/lang/eval-fail-sort-2.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-2.nix create mode 100644 tests/functional/lang/eval-fail-sort-3.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-3.nix create mode 100644 tests/functional/lang/eval-fail-sort-4.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-4.nix create mode 100644 tests/functional/lang/eval-fail-sort-5.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-5.nix create mode 100644 tests/functional/lang/eval-fail-sort-6.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-6.nix create mode 100644 tests/functional/lang/eval-fail-split-1.err.exp create mode 100644 tests/functional/lang/eval-fail-split-1.nix create mode 100644 tests/functional/lang/eval-fail-split-2.err.exp create mode 100644 tests/functional/lang/eval-fail-split-2.nix create mode 100644 tests/functional/lang/eval-fail-split-3.err.exp create mode 100644 tests/functional/lang/eval-fail-split-3.nix create mode 100644 tests/functional/lang/eval-fail-splitVersion-1.err.exp create mode 100644 tests/functional/lang/eval-fail-splitVersion-1.nix create mode 100644 tests/functional/lang/eval-fail-storePath-1.err.exp create mode 100644 tests/functional/lang/eval-fail-storePath-1.nix create mode 100644 tests/functional/lang/eval-fail-stringLength-1.err.exp create mode 100644 tests/functional/lang/eval-fail-stringLength-1.nix create mode 100644 tests/functional/lang/eval-fail-sub-1.err.exp create mode 100644 tests/functional/lang/eval-fail-sub-1.nix create mode 100644 tests/functional/lang/eval-fail-sub-2.err.exp create mode 100644 tests/functional/lang/eval-fail-sub-2.nix create mode 100644 tests/functional/lang/eval-fail-substring-1.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-1.nix create mode 100644 tests/functional/lang/eval-fail-substring-2.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-2.nix create mode 100644 tests/functional/lang/eval-fail-substring-3.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-3.nix create mode 100644 tests/functional/lang/eval-fail-substring-4.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-4.nix create mode 100644 tests/functional/lang/eval-fail-tail-1.err.exp create mode 100644 tests/functional/lang/eval-fail-tail-1.nix create mode 100644 tests/functional/lang/eval-fail-tail-2.err.exp create mode 100644 tests/functional/lang/eval-fail-tail-2.nix create mode 100644 tests/functional/lang/eval-fail-toPath-1.err.exp create mode 100644 tests/functional/lang/eval-fail-toPath-1.nix create mode 100644 tests/functional/lang/eval-fail-toPath-2.err.exp create mode 100644 tests/functional/lang/eval-fail-toPath-2.nix create mode 100644 tests/functional/lang/eval-fail-toString-1.err.exp create mode 100644 tests/functional/lang/eval-fail-toString-1.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-1.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-2.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-3.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-4.nix diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index e722cc48499a..9f2d1f92fa3a 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -54,1270 +54,4 @@ TEST_F(ErrorTraceTest, NestedThrows) } } -#define ASSERT_TRACE1(args, type, message) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE2(args, type, message, context) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE3(args, type, message, context1, context2) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context3)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -// We assume that expr starts with "builtins.derivationStrict { name =", -// otherwise the name attribute position (1, 29) would be invalid. -#define DERIVATION_TRACE_HINTFMT(name) \ - HintFmt( \ - "while evaluating derivation '%s'\n" \ - " whose name attribute is located at %s", \ - name, \ - Pos(1, 29, Pos::String{.source = make_ref(expr)})) - -// To keep things simple, we also assume that derivation name is "foo". -#define ASSERT_DERIVATION_TRACE1(args, type, message) \ - ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ - ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ - ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) - -TEST_F(ErrorTraceTest, replaceStrings) -{ - ASSERT_TRACE2( - "replaceStrings 0 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [] 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); - - ASSERT_TRACE1( - "replaceStrings [ 0 ] [] {}", - EvalError, - HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); - - ASSERT_TRACE2( - "replaceStrings [ 1 ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"oo\" ] [ true ] \"foo\"", - TypeError, - HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"old\" ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); -} - -TEST_F(ErrorTraceTest, scopedImport) {} - -TEST_F(ErrorTraceTest, import) {} - -TEST_F(ErrorTraceTest, typeOf) {} - -TEST_F(ErrorTraceTest, isNull) {} - -TEST_F(ErrorTraceTest, isFunction) {} - -TEST_F(ErrorTraceTest, isInt) {} - -TEST_F(ErrorTraceTest, isFloat) {} - -TEST_F(ErrorTraceTest, isString) {} - -TEST_F(ErrorTraceTest, isBool) {} - -TEST_F(ErrorTraceTest, isPath) {} - -TEST_F(ErrorTraceTest, break) {} - -TEST_F(ErrorTraceTest, abort) {} - -TEST_F(ErrorTraceTest, throw) {} - -TEST_F(ErrorTraceTest, addErrorContext) {} - -TEST_F(ErrorTraceTest, ceil) -{ - ASSERT_TRACE2( - "ceil \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.ceil")); -} - -TEST_F(ErrorTraceTest, floor) -{ - ASSERT_TRACE2( - "floor \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.floor")); -} - -TEST_F(ErrorTraceTest, tryEval) {} - -TEST_F(ErrorTraceTest, getEnv) -{ - ASSERT_TRACE2( - "getEnv [ ]", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getEnv")); -} - -TEST_F(ErrorTraceTest, seq) {} - -TEST_F(ErrorTraceTest, deepSeq) {} - -TEST_F(ErrorTraceTest, trace) {} - -TEST_F(ErrorTraceTest, placeholder) -{ - ASSERT_TRACE2( - "placeholder []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.placeholder")); -} - -TEST_F(ErrorTraceTest, toPath) -{ - ASSERT_TRACE2( - "toPath []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.toPath")); - - ASSERT_TRACE2( - "toPath \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the first argument passed to builtins.toPath")); -} - -TEST_F(ErrorTraceTest, storePath) -{ - ASSERT_TRACE2( - "storePath true", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); -} - -TEST_F(ErrorTraceTest, pathExists) -{ - ASSERT_TRACE2( - "pathExists []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while realising the context of a path")); - - ASSERT_TRACE2( - "pathExists \"zorglub\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), - HintFmt("while realising the context of a path")); -} - -TEST_F(ErrorTraceTest, baseNameOf) -{ - ASSERT_TRACE2( - "baseNameOf []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); -} - -TEST_F(ErrorTraceTest, dirOf) {} - -TEST_F(ErrorTraceTest, readFile) {} - -TEST_F(ErrorTraceTest, findFile) {} - -TEST_F(ErrorTraceTest, hashFile) {} - -TEST_F(ErrorTraceTest, readDir) {} - -TEST_F(ErrorTraceTest, toXML) {} - -TEST_F(ErrorTraceTest, toJSON) {} - -TEST_F(ErrorTraceTest, fromJSON) {} - -TEST_F(ErrorTraceTest, toFile) {} - -TEST_F(ErrorTraceTest, filterSource) -{ - ASSERT_TRACE2( - "filterSource [] []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] ./.", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.filterSource")); - - // Unsupported by store "dummy" - - // ASSERT_TRACE2("filterSource (_: 1) ./.", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while adding path '/home/layus/projects/nix'")); - - // ASSERT_TRACE2("filterSource (_: _: 1) ./.", - // TypeError, - // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), - // HintFmt("while evaluating the return value of the path filter function")); -} - -TEST_F(ErrorTraceTest, path) {} - -TEST_F(ErrorTraceTest, attrNames) -{ - ASSERT_TRACE2( - "attrNames []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrNames")); -} - -TEST_F(ErrorTraceTest, attrValues) -{ - ASSERT_TRACE2( - "attrValues []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrValues")); -} - -TEST_F(ErrorTraceTest, getAttr) -{ - ASSERT_TRACE2( - "getAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" {}", - TypeError, - HintFmt("attribute '%s' missing", "foo"), - HintFmt("in the attribute set under consideration")); -} - -TEST_F(ErrorTraceTest, unsafeGetAttrPos) {} - -TEST_F(ErrorTraceTest, hasAttr) -{ - ASSERT_TRACE2( - "hasAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.hasAttr")); - - ASSERT_TRACE2( - "hasAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.hasAttr")); -} - -TEST_F(ErrorTraceTest, isAttrs) {} - -TEST_F(ErrorTraceTest, removeAttrs) -{ - ASSERT_TRACE2( - "removeAttrs \"\" \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ \"1\" ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); -} - -TEST_F(ErrorTraceTest, listToAttrs) -{ - ASSERT_TRACE2( - "listToAttrs 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the argument passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ {} ]", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in a {name=...; value=...;} pair")); - - ASSERT_TRACE2( - "listToAttrs [ { name = 1; } ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ { name = \"foo\"; } ]", - TypeError, - HintFmt("attribute '%s' missing", "value"), - HintFmt("in a {name=...; value=...;} pair")); -} - -TEST_F(ErrorTraceTest, intersectAttrs) -{ - ASSERT_TRACE2( - "intersectAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); - - ASSERT_TRACE2( - "intersectAttrs {} []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); -} - -TEST_F(ErrorTraceTest, catAttrs) -{ - ASSERT_TRACE2( - "catAttrs [] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); -} - -TEST_F(ErrorTraceTest, functionArgs) -{ - ASSERT_TRACE1("functionArgs {}", TypeError, HintFmt("'functionArgs' requires a function")); -} - -TEST_F(ErrorTraceTest, mapAttrs) -{ - ASSERT_TRACE2( - "mapAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); - - // XXX: deferred - // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", - // TypeError, - // HintFmt("cannot coerce %s to a string", "an integer"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, zipAttrsWith) -{ - ASSERT_TRACE2( - "zipAttrsWith [] [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); - - ASSERT_TRACE2( - "zipAttrsWith (_: 1) [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); - - // XXX: How to properly tell that the function takes two arguments ? - // The same question also applies to sort, and maybe others. - // Due to laziness, we only create a thunk, and it fails later on. - // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while evaluating the attribute 'foo'")); - - // XXX: Also deferred deeply - // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", - // TypeError, - // HintFmt("cannot coerce %s to a string", "a list"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, isList) {} - -TEST_F(ErrorTraceTest, elemAt) -{ - ASSERT_TRACE2( - "elemAt \"foo\" (-1)", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); - - ASSERT_TRACE1( - "elemAt [] (-1)", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); - - ASSERT_TRACE1( - "elemAt [\"foo\"] 3", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); -} - -TEST_F(ErrorTraceTest, head) -{ - ASSERT_TRACE2( - "head 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.head'")); - - ASSERT_TRACE1("head []", Error, HintFmt("'builtins.head' called on an empty list")); -} - -TEST_F(ErrorTraceTest, tail) -{ - ASSERT_TRACE2( - "tail 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.tail'")); - - ASSERT_TRACE1("tail []", Error, HintFmt("'builtins.tail' called on an empty list")); -} - -TEST_F(ErrorTraceTest, map) -{ - ASSERT_TRACE2( - "map 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.map")); - - ASSERT_TRACE2( - "map 1 [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.map")); -} - -TEST_F(ErrorTraceTest, filter) -{ - ASSERT_TRACE2( - "filter 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter (_: 5) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); -} - -TEST_F(ErrorTraceTest, elem) -{ - ASSERT_TRACE2( - "elem 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.elem")); -} - -TEST_F(ErrorTraceTest, concatLists) -{ - ASSERT_TRACE2( - "concatLists 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ 1 ]", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ [1] \"foo\" ]", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); -} - -TEST_F(ErrorTraceTest, length) -{ - ASSERT_TRACE2( - "length 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); - - ASSERT_TRACE2( - "length \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); -} - -TEST_F(ErrorTraceTest, foldlPrime) -{ - ASSERT_TRACE2( - "foldl' 1 \"foo\" true", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); - - ASSERT_TRACE2( - "foldl' (_: 1) \"foo\" true", - TypeError, - HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); - - ASSERT_TRACE1( - "foldl' (_: 1) \"foo\" [ true ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "foldl' (a: b: a && b) \"foo\" [ true ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("in the left operand of the AND (&&) operator")); -} - -TEST_F(ErrorTraceTest, any) -{ - ASSERT_TRACE2( - "any 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.any")); -} - -TEST_F(ErrorTraceTest, all) -{ - ASSERT_TRACE2( - "all 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.all")); -} - -TEST_F(ErrorTraceTest, genList) -{ - ASSERT_TRACE2( - "genList 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.genList")); - - ASSERT_TRACE2( - "genList 1 2", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.genList")); - - // XXX: deferred - // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", - // TypeError, - // HintFmt("cannot add %s to an integer", "a string"), - // HintFmt("while evaluating anonymous lambda")); - - ASSERT_TRACE1("genList false (-3)", EvalError, HintFmt("cannot create list of size %d", -3)); -} - -TEST_F(ErrorTraceTest, sort) -{ - ASSERT_TRACE2( - "sort 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.sort")); - - ASSERT_TRACE2( - "sort 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.sort")); - - ASSERT_TRACE1( - "sort (_: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "sort (_: _: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); - - // XXX: Trace too deep, need better asserts - // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s", "a string", "a set")); - - // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); -} - -TEST_F(ErrorTraceTest, partition) -{ - ASSERT_TRACE2( - "partition 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); -} - -TEST_F(ErrorTraceTest, groupBy) -{ - ASSERT_TRACE2( - "groupBy 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (x: x) [ \"foo\" \"bar\" 1 ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); -} - -TEST_F(ErrorTraceTest, concatMap) -{ - ASSERT_TRACE2( - "concatMap 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) [ \"foo\" ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: \"foo\") [ 1 2 ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); -} - -TEST_F(ErrorTraceTest, add) -{ - ASSERT_TRACE2( - "add \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the addition")); - - ASSERT_TRACE2( - "add 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the addition")); -} - -TEST_F(ErrorTraceTest, sub) -{ - ASSERT_TRACE2( - "sub \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the subtraction")); - - ASSERT_TRACE2( - "sub 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the subtraction")); -} - -TEST_F(ErrorTraceTest, mul) -{ - ASSERT_TRACE2( - "mul \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the multiplication")); - - ASSERT_TRACE2( - "mul 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the multiplication")); -} - -TEST_F(ErrorTraceTest, div) -{ - ASSERT_TRACE2( - "div \"foo\" 1 # TODO: an integer was expected -> a number", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first operand of the division")); - - ASSERT_TRACE2( - "div 1 \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second operand of the division")); - - ASSERT_TRACE1("div \"foo\" 0", EvalError, HintFmt("division by zero")); -} - -TEST_F(ErrorTraceTest, bitAnd) -{ - ASSERT_TRACE2( - "bitAnd 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitAnd")); - - ASSERT_TRACE2( - "bitAnd 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitAnd")); -} - -TEST_F(ErrorTraceTest, bitOr) -{ - ASSERT_TRACE2( - "bitOr 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitOr")); - - ASSERT_TRACE2( - "bitOr 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitOr")); -} - -TEST_F(ErrorTraceTest, bitXor) -{ - ASSERT_TRACE2( - "bitXor 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitXor")); - - ASSERT_TRACE2( - "bitXor 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitXor")); -} - -TEST_F(ErrorTraceTest, lessThan) -{ - ASSERT_TRACE1( - "lessThan 1 \"foo\"", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL))); - - ASSERT_TRACE1( - "lessThan {} {}", - EvalError, - HintFmt( - "cannot compare %s with %s; values of that type are incomparable (values are %s and %s)", - "a set", - "a set", - Uncolored("{ }"), - Uncolored("{ }"))); - - ASSERT_TRACE2( - "lessThan [ 1 2 ] [ \"foo\" ]", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while comparing two list elements")); -} - -TEST_F(ErrorTraceTest, toString) -{ - ASSERT_TRACE2( - "toString { a = 1; }", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), - HintFmt("while evaluating the first argument passed to builtins.toString")); -} - -TEST_F(ErrorTraceTest, substring) -{ - ASSERT_TRACE2( - "substring {} \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 3 \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 0 3 {}", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); - - ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, HintFmt("negative start position in 'substring'")); -} - -TEST_F(ErrorTraceTest, stringLength) -{ - ASSERT_TRACE2( - "stringLength {} # TODO: context is missing ???", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the argument passed to builtins.stringLength")); -} - -TEST_F(ErrorTraceTest, hashString) -{ - ASSERT_TRACE2( - "hashString 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.hashString")); - - ASSERT_TRACE1( - "hashString \"foo\" \"content\"", - UsageError, - HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); - - ASSERT_TRACE2( - "hashString \"sha256\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.hashString")); -} - -TEST_F(ErrorTraceTest, match) -{ - ASSERT_TRACE2( - "match 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.match")); - - ASSERT_TRACE2( - "match \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.match")); - - ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, HintFmt("invalid regular expression '%s'", "(.*")); -} - -TEST_F(ErrorTraceTest, split) -{ - ASSERT_TRACE2( - "split 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.split")); - - ASSERT_TRACE2( - "split \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.split")); - - ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, HintFmt("invalid regular expression '%s'", "f(o*o")); -} - -TEST_F(ErrorTraceTest, concatStringsSep) -{ - ASSERT_TRACE2( - "concatStringsSep 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt( - "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); -} - -TEST_F(ErrorTraceTest, parseDrvName) -{ - ASSERT_TRACE2( - "parseDrvName 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); -} - -TEST_F(ErrorTraceTest, compareVersions) -{ - ASSERT_TRACE2( - "compareVersions 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.compareVersions")); - - ASSERT_TRACE2( - "compareVersions \"abd\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.compareVersions")); -} - -TEST_F(ErrorTraceTest, splitVersion) -{ - ASSERT_TRACE2( - "splitVersion 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.splitVersion")); -} - -TEST_F(ErrorTraceTest, traceVerbose) {} - -TEST_F(ErrorTraceTest, derivationStrict) -{ - ASSERT_TRACE2( - "derivationStrict \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", "\"\""), - HintFmt("while evaluating the argument passed to builtins.derivationStrict")); - - ASSERT_TRACE2( - "derivationStrict {}", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in the attrset passed as argument to builtins.derivationStrict")); - - ASSERT_TRACE3( - "derivationStrict { name = 1; }", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), - HintFmt("while evaluating the derivation attribute 'name'")); - - ASSERT_DERIVATION_TRACE1( - "derivationStrict { name = \"foo\"; }", EvalError, HintFmt("required attribute 'builder' missing")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", - EvalError, - HintFmt("expected a list but found %s: %s", "a string", "\"out\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", - EvalError, - HintFmt("derivation cannot have an empty set of outputs"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", - EvalError, - HintFmt("duplicate derivation output '%s'", "out"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); -} - } /* namespace nix */ diff --git a/tests/functional/lang/eval-fail-add-1.err.exp b/tests/functional/lang/eval-fail-add-1.err.exp new file mode 100644 index 000000000000..4a72eba125e7 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'add' builtin + at /pwd/lang/eval-fail-add-1.nix:1:1: + 1| builtins.add "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the addition + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-add-1.nix b/tests/functional/lang/eval-fail-add-1.nix new file mode 100644 index 000000000000..4a59d94e68af --- /dev/null +++ b/tests/functional/lang/eval-fail-add-1.nix @@ -0,0 +1 @@ +builtins.add "foo" 1 diff --git a/tests/functional/lang/eval-fail-add-2.err.exp b/tests/functional/lang/eval-fail-add-2.err.exp new file mode 100644 index 000000000000..429636348d27 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'add' builtin + at /pwd/lang/eval-fail-add-2.nix:1:1: + 1| builtins.add 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the addition + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-add-2.nix b/tests/functional/lang/eval-fail-add-2.nix new file mode 100644 index 000000000000..1b785aba2700 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-2.nix @@ -0,0 +1 @@ +builtins.add 1 "foo" diff --git a/tests/functional/lang/eval-fail-all-1.err.exp b/tests/functional/lang/eval-fail-all-1.err.exp new file mode 100644 index 000000000000..95e8735f8ca5 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-1.nix:1:1: + 1| builtins.all 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.all + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-all-1.nix b/tests/functional/lang/eval-fail-all-1.nix new file mode 100644 index 000000000000..c1ae6ff5e18c --- /dev/null +++ b/tests/functional/lang/eval-fail-all-1.nix @@ -0,0 +1 @@ +builtins.all 1 "foo" diff --git a/tests/functional/lang/eval-fail-all-2.err.exp b/tests/functional/lang/eval-fail-all-2.err.exp new file mode 100644 index 000000000000..584543f5e85e --- /dev/null +++ b/tests/functional/lang/eval-fail-all-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-2.nix:1:1: + 1| builtins.all (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.all + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-all-2.nix b/tests/functional/lang/eval-fail-all-2.nix new file mode 100644 index 000000000000..b8ec8c87125c --- /dev/null +++ b/tests/functional/lang/eval-fail-all-2.nix @@ -0,0 +1 @@ +builtins.all (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-all-3.err.exp b/tests/functional/lang/eval-fail-all-3.err.exp new file mode 100644 index 000000000000..692cebe3c214 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-3.nix:1:1: + 1| builtins.all (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.all + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-all-3.nix b/tests/functional/lang/eval-fail-all-3.nix new file mode 100644 index 000000000000..68f6ce6567e5 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-3.nix @@ -0,0 +1 @@ +builtins.all (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-any-1.err.exp b/tests/functional/lang/eval-fail-any-1.err.exp new file mode 100644 index 000000000000..2d81dae42547 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-1.nix:1:1: + 1| builtins.any 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.any + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-any-1.nix b/tests/functional/lang/eval-fail-any-1.nix new file mode 100644 index 000000000000..aa59791b73fe --- /dev/null +++ b/tests/functional/lang/eval-fail-any-1.nix @@ -0,0 +1 @@ +builtins.any 1 "foo" diff --git a/tests/functional/lang/eval-fail-any-2.err.exp b/tests/functional/lang/eval-fail-any-2.err.exp new file mode 100644 index 000000000000..a03a9e7c34e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-2.nix:1:1: + 1| builtins.any (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.any + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-any-2.nix b/tests/functional/lang/eval-fail-any-2.nix new file mode 100644 index 000000000000..64a5d3acc801 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-2.nix @@ -0,0 +1 @@ +builtins.any (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-any-3.err.exp b/tests/functional/lang/eval-fail-any-3.err.exp new file mode 100644 index 000000000000..b06d27abdf65 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-3.nix:1:1: + 1| builtins.any (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.any + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-any-3.nix b/tests/functional/lang/eval-fail-any-3.nix new file mode 100644 index 000000000000..906d08252aa8 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-3.nix @@ -0,0 +1 @@ +builtins.any (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-attrNames-1.err.exp b/tests/functional/lang/eval-fail-attrNames-1.err.exp new file mode 100644 index 000000000000..46f71a49102d --- /dev/null +++ b/tests/functional/lang/eval-fail-attrNames-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'attrNames' builtin + at /pwd/lang/eval-fail-attrNames-1.nix:1:1: + 1| builtins.attrNames [ ] + | ^ + 2| + + … while evaluating the argument passed to builtins.attrNames + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-attrNames-1.nix b/tests/functional/lang/eval-fail-attrNames-1.nix new file mode 100644 index 000000000000..38f68e4d67b1 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrNames-1.nix @@ -0,0 +1 @@ +builtins.attrNames [ ] diff --git a/tests/functional/lang/eval-fail-attrValues-1.err.exp b/tests/functional/lang/eval-fail-attrValues-1.err.exp new file mode 100644 index 000000000000..c7018784f1b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrValues-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'attrValues' builtin + at /pwd/lang/eval-fail-attrValues-1.nix:1:1: + 1| builtins.attrValues [ ] + | ^ + 2| + + … while evaluating the argument passed to builtins.attrValues + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-attrValues-1.nix b/tests/functional/lang/eval-fail-attrValues-1.nix new file mode 100644 index 000000000000..c3154854c78f --- /dev/null +++ b/tests/functional/lang/eval-fail-attrValues-1.nix @@ -0,0 +1 @@ +builtins.attrValues [ ] diff --git a/tests/functional/lang/eval-fail-baseNameOf-1.err.exp b/tests/functional/lang/eval-fail-baseNameOf-1.err.exp new file mode 100644 index 000000000000..4116a1a06391 --- /dev/null +++ b/tests/functional/lang/eval-fail-baseNameOf-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'baseNameOf' builtin + at /pwd/lang/eval-fail-baseNameOf-1.nix:1:1: + 1| builtins.baseNameOf [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.baseNameOf + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-baseNameOf-1.nix b/tests/functional/lang/eval-fail-baseNameOf-1.nix new file mode 100644 index 000000000000..3099c87f5714 --- /dev/null +++ b/tests/functional/lang/eval-fail-baseNameOf-1.nix @@ -0,0 +1 @@ +builtins.baseNameOf [ ] diff --git a/tests/functional/lang/eval-fail-bitAnd-1.err.exp b/tests/functional/lang/eval-fail-bitAnd-1.err.exp new file mode 100644 index 000000000000..04ae5c85400b --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitAnd' builtin + at /pwd/lang/eval-fail-bitAnd-1.nix:1:1: + 1| builtins.bitAnd 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitAnd + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitAnd-1.nix b/tests/functional/lang/eval-fail-bitAnd-1.nix new file mode 100644 index 000000000000..24c5d60bdd02 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-1.nix @@ -0,0 +1 @@ +builtins.bitAnd 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitAnd-2.err.exp b/tests/functional/lang/eval-fail-bitAnd-2.err.exp new file mode 100644 index 000000000000..f845462f1262 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitAnd' builtin + at /pwd/lang/eval-fail-bitAnd-2.nix:1:1: + 1| builtins.bitAnd 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitAnd + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitAnd-2.nix b/tests/functional/lang/eval-fail-bitAnd-2.nix new file mode 100644 index 000000000000..12ea1451f582 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-2.nix @@ -0,0 +1 @@ +builtins.bitAnd 1 2.2 diff --git a/tests/functional/lang/eval-fail-bitOr-1.err.exp b/tests/functional/lang/eval-fail-bitOr-1.err.exp new file mode 100644 index 000000000000..38b498da4697 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitOr' builtin + at /pwd/lang/eval-fail-bitOr-1.nix:1:1: + 1| builtins.bitOr 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitOr + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitOr-1.nix b/tests/functional/lang/eval-fail-bitOr-1.nix new file mode 100644 index 000000000000..2eab4871ca65 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-1.nix @@ -0,0 +1 @@ +builtins.bitOr 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitOr-2.err.exp b/tests/functional/lang/eval-fail-bitOr-2.err.exp new file mode 100644 index 000000000000..a1ca655add7a --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitOr' builtin + at /pwd/lang/eval-fail-bitOr-2.nix:1:1: + 1| builtins.bitOr 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitOr + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitOr-2.nix b/tests/functional/lang/eval-fail-bitOr-2.nix new file mode 100644 index 000000000000..f66e3a3635ab --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-2.nix @@ -0,0 +1 @@ +builtins.bitOr 1 2.2 diff --git a/tests/functional/lang/eval-fail-bitXor-1.err.exp b/tests/functional/lang/eval-fail-bitXor-1.err.exp new file mode 100644 index 000000000000..f051ef622069 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitXor' builtin + at /pwd/lang/eval-fail-bitXor-1.nix:1:1: + 1| builtins.bitXor 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitXor + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitXor-1.nix b/tests/functional/lang/eval-fail-bitXor-1.nix new file mode 100644 index 000000000000..c1b0f1b4e82d --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-1.nix @@ -0,0 +1 @@ +builtins.bitXor 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitXor-2.err.exp b/tests/functional/lang/eval-fail-bitXor-2.err.exp new file mode 100644 index 000000000000..3b5ab1317ab0 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitXor' builtin + at /pwd/lang/eval-fail-bitXor-2.nix:1:1: + 1| builtins.bitXor 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitXor + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitXor-2.nix b/tests/functional/lang/eval-fail-bitXor-2.nix new file mode 100644 index 000000000000..b9f7ad9c2890 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-2.nix @@ -0,0 +1 @@ +builtins.bitXor 1 2.2 diff --git a/tests/functional/lang/eval-fail-catAttrs-1.err.exp b/tests/functional/lang/eval-fail-catAttrs-1.err.exp new file mode 100644 index 000000000000..e669dc39a6e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-1.nix:1:1: + 1| builtins.catAttrs [ ] { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.catAttrs + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-catAttrs-1.nix b/tests/functional/lang/eval-fail-catAttrs-1.nix new file mode 100644 index 000000000000..97d06f5b63be --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-1.nix @@ -0,0 +1 @@ +builtins.catAttrs [ ] { } diff --git a/tests/functional/lang/eval-fail-catAttrs-2.err.exp b/tests/functional/lang/eval-fail-catAttrs-2.err.exp new file mode 100644 index 000000000000..2f6c1425d6a6 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-2.nix:1:1: + 1| builtins.catAttrs "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.catAttrs + + error: expected a list but found a set: { } diff --git a/tests/functional/lang/eval-fail-catAttrs-2.nix b/tests/functional/lang/eval-fail-catAttrs-2.nix new file mode 100644 index 000000000000..caf616b20b84 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-2.nix @@ -0,0 +1 @@ +builtins.catAttrs "foo" { } diff --git a/tests/functional/lang/eval-fail-catAttrs-3.err.exp b/tests/functional/lang/eval-fail-catAttrs-3.err.exp new file mode 100644 index 000000000000..2643d37f9e96 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-3.nix:1:1: + 1| builtins.catAttrs "foo" [ 1 ] + | ^ + 2| + + … while evaluating an element in the list passed as second argument to builtins.catAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-catAttrs-3.nix b/tests/functional/lang/eval-fail-catAttrs-3.nix new file mode 100644 index 000000000000..82595b94ed0f --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-3.nix @@ -0,0 +1 @@ +builtins.catAttrs "foo" [ 1 ] diff --git a/tests/functional/lang/eval-fail-catAttrs-4.err.exp b/tests/functional/lang/eval-fail-catAttrs-4.err.exp new file mode 100644 index 000000000000..ffd5fcec7c0b --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-4.nix:1:1: + 1| builtins.catAttrs "foo" [ + | ^ + 2| { foo = 1; } + + … while evaluating an element in the list passed as second argument to builtins.catAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-catAttrs-4.nix b/tests/functional/lang/eval-fail-catAttrs-4.nix new file mode 100644 index 000000000000..1f294a693c6a --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-4.nix @@ -0,0 +1,5 @@ +builtins.catAttrs "foo" [ + { foo = 1; } + 1 + { bar = 5; } +] diff --git a/tests/functional/lang/eval-fail-ceil-1.err.exp b/tests/functional/lang/eval-fail-ceil-1.err.exp new file mode 100644 index 000000000000..96d8b1df7a8e --- /dev/null +++ b/tests/functional/lang/eval-fail-ceil-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'ceil' builtin + at /pwd/lang/eval-fail-ceil-1.nix:1:1: + 1| builtins.ceil "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.ceil + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-ceil-1.nix b/tests/functional/lang/eval-fail-ceil-1.nix new file mode 100644 index 000000000000..0ea67754c358 --- /dev/null +++ b/tests/functional/lang/eval-fail-ceil-1.nix @@ -0,0 +1 @@ +builtins.ceil "foo" diff --git a/tests/functional/lang/eval-fail-compareVersions-1.err.exp b/tests/functional/lang/eval-fail-compareVersions-1.err.exp new file mode 100644 index 000000000000..8632cf237e1b --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'compareVersions' builtin + at /pwd/lang/eval-fail-compareVersions-1.nix:1:1: + 1| builtins.compareVersions 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.compareVersions + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-compareVersions-1.nix b/tests/functional/lang/eval-fail-compareVersions-1.nix new file mode 100644 index 000000000000..f408381cf4c5 --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-1.nix @@ -0,0 +1 @@ +builtins.compareVersions 1 { } diff --git a/tests/functional/lang/eval-fail-compareVersions-2.err.exp b/tests/functional/lang/eval-fail-compareVersions-2.err.exp new file mode 100644 index 000000000000..bcd246460f2f --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'compareVersions' builtin + at /pwd/lang/eval-fail-compareVersions-2.nix:1:1: + 1| builtins.compareVersions "abd" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.compareVersions + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-compareVersions-2.nix b/tests/functional/lang/eval-fail-compareVersions-2.nix new file mode 100644 index 000000000000..49c9f0a8874e --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-2.nix @@ -0,0 +1 @@ +builtins.compareVersions "abd" { } diff --git a/tests/functional/lang/eval-fail-concatLists-1.err.exp b/tests/functional/lang/eval-fail-concatLists-1.err.exp new file mode 100644 index 000000000000..15193daae1ca --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-1.nix:1:1: + 1| builtins.concatLists 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.concatLists + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatLists-1.nix b/tests/functional/lang/eval-fail-concatLists-1.nix new file mode 100644 index 000000000000..97e171cb6fa4 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-1.nix @@ -0,0 +1 @@ +builtins.concatLists 1 diff --git a/tests/functional/lang/eval-fail-concatLists-2.err.exp b/tests/functional/lang/eval-fail-concatLists-2.err.exp new file mode 100644 index 000000000000..35fbc09bfe35 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-2.nix:1:1: + 1| builtins.concatLists [ 1 ] + | ^ + 2| + + … while evaluating a value of the list passed to builtins.concatLists + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatLists-2.nix b/tests/functional/lang/eval-fail-concatLists-2.nix new file mode 100644 index 000000000000..0ad98a8eb4ac --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-2.nix @@ -0,0 +1 @@ +builtins.concatLists [ 1 ] diff --git a/tests/functional/lang/eval-fail-concatLists-3.err.exp b/tests/functional/lang/eval-fail-concatLists-3.err.exp new file mode 100644 index 000000000000..d762eb55bdaa --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-3.nix:1:1: + 1| builtins.concatLists [ + | ^ + 2| [ 1 ] + + … while evaluating a value of the list passed to builtins.concatLists + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatLists-3.nix b/tests/functional/lang/eval-fail-concatLists-3.nix new file mode 100644 index 000000000000..2366f314e1b7 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-3.nix @@ -0,0 +1,4 @@ +builtins.concatLists [ + [ 1 ] + "foo" +] diff --git a/tests/functional/lang/eval-fail-concatMap-1.err.exp b/tests/functional/lang/eval-fail-concatMap-1.err.exp new file mode 100644 index 000000000000..e43c160a38e2 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-1.nix:1:1: + 1| builtins.concatMap 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.concatMap + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatMap-1.nix b/tests/functional/lang/eval-fail-concatMap-1.nix new file mode 100644 index 000000000000..68c9c3560f6a --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-1.nix @@ -0,0 +1 @@ +builtins.concatMap 1 "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-2.err.exp b/tests/functional/lang/eval-fail-concatMap-2.err.exp new file mode 100644 index 000000000000..52c51fd0157b --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-2.nix:1:1: + 1| builtins.concatMap (x: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.concatMap + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-2.nix b/tests/functional/lang/eval-fail-concatMap-2.nix new file mode 100644 index 000000000000..b98860470caa --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-2.nix @@ -0,0 +1 @@ +builtins.concatMap (x: 1) "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-3.err.exp b/tests/functional/lang/eval-fail-concatMap-3.err.exp new file mode 100644 index 000000000000..21d13118f0cd --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-3.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-3.nix:1:1: + 1| builtins.concatMap (x: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.concatMap + at /pwd/lang/eval-fail-concatMap-3.nix:1:21: + 1| builtins.concatMap (x: 1) [ "foo" ] + | ^ + 2| + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatMap-3.nix b/tests/functional/lang/eval-fail-concatMap-3.nix new file mode 100644 index 000000000000..f4a519060358 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-3.nix @@ -0,0 +1 @@ +builtins.concatMap (x: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-concatMap-4.err.exp b/tests/functional/lang/eval-fail-concatMap-4.err.exp new file mode 100644 index 000000000000..1b8e865fd3a1 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-4.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-4.nix:1:1: + 1| builtins.concatMap (x: "foo") [ + | ^ + 2| 1 + + … while evaluating the return value of the function passed to builtins.concatMap + at /pwd/lang/eval-fail-concatMap-4.nix:1:21: + 1| builtins.concatMap (x: "foo") [ + | ^ + 2| 1 + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-4.nix b/tests/functional/lang/eval-fail-concatMap-4.nix new file mode 100644 index 000000000000..0f678005bced --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-4.nix @@ -0,0 +1,4 @@ +builtins.concatMap (x: "foo") [ + 1 + 2 +] diff --git a/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp new file mode 100644 index 000000000000..4b40db46d681 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-1.nix:1:1: + 1| builtins.concatStringsSep 1 { } + | ^ + 2| + + … while evaluating the first argument (the separator string) passed to builtins.concatStringsSep + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatStringsSep-1.nix b/tests/functional/lang/eval-fail-concatStringsSep-1.nix new file mode 100644 index 000000000000..0c10cea34630 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-1.nix @@ -0,0 +1 @@ +builtins.concatStringsSep 1 { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp new file mode 100644 index 000000000000..9cc806f4955f --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-2.nix:1:1: + 1| builtins.concatStringsSep "foo" { } + | ^ + 2| + + … while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep + + error: expected a list but found a set: { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-2.nix b/tests/functional/lang/eval-fail-concatStringsSep-2.nix new file mode 100644 index 000000000000..ae6a5f1984d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-2.nix @@ -0,0 +1 @@ +builtins.concatStringsSep "foo" { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp new file mode 100644 index 000000000000..5c098d471e72 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-3.nix:1:1: + 1| builtins.concatStringsSep "foo" [ + | ^ + 2| 1 + + … while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep + + error: cannot coerce an integer to a string: 1 diff --git a/tests/functional/lang/eval-fail-concatStringsSep-3.nix b/tests/functional/lang/eval-fail-concatStringsSep-3.nix new file mode 100644 index 000000000000..f51ae1c7ddc5 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-3.nix @@ -0,0 +1,5 @@ +builtins.concatStringsSep "foo" [ + 1 + 2 + { } +] diff --git a/tests/functional/lang/eval-fail-derivationStrict-1.err.exp b/tests/functional/lang/eval-fail-derivationStrict-1.err.exp new file mode 100644 index 000000000000..468c789e8db4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-1.nix:1:1: + 1| builtins.derivationStrict "" + | ^ + 2| + + … while evaluating the argument passed to builtins.derivationStrict + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-derivationStrict-1.nix b/tests/functional/lang/eval-fail-derivationStrict-1.nix new file mode 100644 index 000000000000..948230469d98 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-1.nix @@ -0,0 +1 @@ +builtins.derivationStrict "" diff --git a/tests/functional/lang/eval-fail-derivationStrict-10.err.exp b/tests/functional/lang/eval-fail-derivationStrict-10.err.exp new file mode 100644 index 000000000000..21eca42fad06 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-10.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-10.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-10.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-10.nix:5:3: + 4| system = 1; + 5| outputs = { }; + | ^ + 6| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-10.nix b/tests/functional/lang/eval-fail-derivationStrict-10.nix new file mode 100644 index 000000000000..410f48a46537 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-10.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = { }; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-11.err.exp b/tests/functional/lang/eval-fail-derivationStrict-11.err.exp new file mode 100644 index 000000000000..b0fe664e5202 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-11.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-11.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-11.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-11.nix:5:3: + 4| system = 1; + 5| outputs = "drvPath"; + | ^ + 6| } + + error: invalid derivation output name 'drvPath' diff --git a/tests/functional/lang/eval-fail-derivationStrict-11.nix b/tests/functional/lang/eval-fail-derivationStrict-11.nix new file mode 100644 index 000000000000..60947b6e2ee6 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-11.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "drvPath"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-12.err.exp b/tests/functional/lang/eval-fail-derivationStrict-12.err.exp new file mode 100644 index 000000000000..adfc0064bfb7 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-12.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-12.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-12.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-12.nix:3:3: + 2| name = "foo"; + 3| outputs = "out"; + | ^ + 4| __structuredAttrs = true; + + error: expected a list but found a string: "out" diff --git a/tests/functional/lang/eval-fail-derivationStrict-12.nix b/tests/functional/lang/eval-fail-derivationStrict-12.nix new file mode 100644 index 000000000000..516b8ba39ace --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-12.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + outputs = "out"; + __structuredAttrs = true; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-13.err.exp b/tests/functional/lang/eval-fail-derivationStrict-13.err.exp new file mode 100644 index 000000000000..8a313333d58f --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-13.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-13.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-13.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-13.nix:5:3: + 4| system = 1; + 5| outputs = [ ]; + | ^ + 6| } + + error: derivation cannot have an empty set of outputs diff --git a/tests/functional/lang/eval-fail-derivationStrict-13.nix b/tests/functional/lang/eval-fail-derivationStrict-13.nix new file mode 100644 index 000000000000..9a104d2253fd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-13.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-14.err.exp b/tests/functional/lang/eval-fail-derivationStrict-14.err.exp new file mode 100644 index 000000000000..fb0c5423c4dc --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-14.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-14.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-14.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-14.nix:5:3: + 4| system = 1; + 5| outputs = [ "drvPath" ]; + | ^ + 6| } + + error: invalid derivation output name 'drvPath' diff --git a/tests/functional/lang/eval-fail-derivationStrict-14.nix b/tests/functional/lang/eval-fail-derivationStrict-14.nix new file mode 100644 index 000000000000..4d286d34c725 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-14.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ "drvPath" ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-15.err.exp b/tests/functional/lang/eval-fail-derivationStrict-15.err.exp new file mode 100644 index 000000000000..8df047069136 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-15.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-15.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-15.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-15.nix:5:3: + 4| system = 1; + 5| outputs = [ + | ^ + 6| "out" + + error: duplicate derivation output 'out' diff --git a/tests/functional/lang/eval-fail-derivationStrict-15.nix b/tests/functional/lang/eval-fail-derivationStrict-15.nix new file mode 100644 index 000000000000..96e5a9de404b --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-15.nix @@ -0,0 +1,9 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ + "out" + "out" + ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-16.err.exp b/tests/functional/lang/eval-fail-derivationStrict-16.err.exp new file mode 100644 index 000000000000..c969aa30d0d3 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-16.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-16.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-16.nix:2:3 + + … while evaluating attribute '__contentAddressed' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-16.nix:6:3: + 5| outputs = "out"; + 6| __contentAddressed = "true"; + | ^ + 7| } + + error: expected a Boolean but found a string: "true" diff --git a/tests/functional/lang/eval-fail-derivationStrict-16.nix b/tests/functional/lang/eval-fail-derivationStrict-16.nix new file mode 100644 index 000000000000..554ad09df1a2 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-16.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + __contentAddressed = "true"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-17.err.exp b/tests/functional/lang/eval-fail-derivationStrict-17.err.exp new file mode 100644 index 000000000000..ed7558c4cbab --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-17.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-17.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-17.nix:2:3 + + … while evaluating attribute '__impure' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-17.nix:6:3: + 5| outputs = "out"; + 6| __impure = "true"; + | ^ + 7| } + + error: expected a Boolean but found a string: "true" diff --git a/tests/functional/lang/eval-fail-derivationStrict-17.nix b/tests/functional/lang/eval-fail-derivationStrict-17.nix new file mode 100644 index 000000000000..4a357ba12d7f --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-17.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + __impure = "true"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-19.err.exp b/tests/functional/lang/eval-fail-derivationStrict-19.err.exp new file mode 100644 index 000000000000..4861b1cd3abd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-19.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-19.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-19.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-19.nix:6:3: + 5| outputs = "out"; + 6| args = "foo"; + | ^ + 7| } + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-derivationStrict-19.nix b/tests/functional/lang/eval-fail-derivationStrict-19.nix new file mode 100644 index 000000000000..912be12c5708 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-19.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = "foo"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-2.err.exp b/tests/functional/lang/eval-fail-derivationStrict-2.err.exp new file mode 100644 index 000000000000..7ba9fc825595 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-2.nix:1:1: + 1| builtins.derivationStrict { } + | ^ + 2| + + … in the attrset passed as argument to builtins.derivationStrict + + error: attribute 'name' missing diff --git a/tests/functional/lang/eval-fail-derivationStrict-2.nix b/tests/functional/lang/eval-fail-derivationStrict-2.nix new file mode 100644 index 000000000000..70a07b36d2d5 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-2.nix @@ -0,0 +1 @@ +builtins.derivationStrict { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-20.err.exp b/tests/functional/lang/eval-fail-derivationStrict-20.err.exp new file mode 100644 index 000000000000..df1805bef681 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-20.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-20.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-20.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-20.nix:6:3: + 5| outputs = "out"; + 6| args = [ { } ]; + | ^ + 7| } + + … while evaluating an element of the argument list + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-20.nix b/tests/functional/lang/eval-fail-derivationStrict-20.nix new file mode 100644 index 000000000000..17bc3a26c906 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-20.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = [ { } ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-21.err.exp b/tests/functional/lang/eval-fail-derivationStrict-21.err.exp new file mode 100644 index 000000000000..0a6384a77957 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-21.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-21.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-21.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-21.nix:6:3: + 5| outputs = "out"; + 6| args = [ + | ^ + 7| "a" + + … while evaluating an element of the argument list + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-21.nix b/tests/functional/lang/eval-fail-derivationStrict-21.nix new file mode 100644 index 000000000000..7846874194c3 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-21.nix @@ -0,0 +1,10 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = [ + "a" + { } + ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-22.err.exp b/tests/functional/lang/eval-fail-derivationStrict-22.err.exp new file mode 100644 index 000000000000..91a44ce1e425 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-22.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-22.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-22.nix:2:3 + + … while evaluating attribute 'FOO' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-22.nix:6:3: + 5| outputs = "out"; + 6| FOO = { }; + | ^ + 7| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-22.nix b/tests/functional/lang/eval-fail-derivationStrict-22.nix new file mode 100644 index 000000000000..7b3b7c824855 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-22.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + FOO = { }; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-3.err.exp b/tests/functional/lang/eval-fail-derivationStrict-3.err.exp new file mode 100644 index 000000000000..f8c301a97793 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-3.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-3.nix:1:1: + 1| builtins.derivationStrict { name = 1; } + | ^ + 2| + + … while evaluating the derivation attribute 'name' + at /pwd/lang/eval-fail-derivationStrict-3.nix:1:29: + 1| builtins.derivationStrict { name = 1; } + | ^ + 2| + + … while evaluating the `name` attribute passed to builtins.derivationStrict + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-derivationStrict-3.nix b/tests/functional/lang/eval-fail-derivationStrict-3.nix new file mode 100644 index 000000000000..a18edebd8701 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-3.nix @@ -0,0 +1 @@ +builtins.derivationStrict { name = 1; } diff --git a/tests/functional/lang/eval-fail-derivationStrict-4.err.exp b/tests/functional/lang/eval-fail-derivationStrict-4.err.exp new file mode 100644 index 000000000000..b598f0f11ccd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-4.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-4.nix:1:1: + 1| builtins.derivationStrict { name = "foo"; } + | ^ + 2| + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-4.nix:1:29 + + error: required attribute 'builder' missing diff --git a/tests/functional/lang/eval-fail-derivationStrict-4.nix b/tests/functional/lang/eval-fail-derivationStrict-4.nix new file mode 100644 index 000000000000..f13692bc5e10 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-4.nix @@ -0,0 +1 @@ +builtins.derivationStrict { name = "foo"; } diff --git a/tests/functional/lang/eval-fail-derivationStrict-5.err.exp b/tests/functional/lang/eval-fail-derivationStrict-5.err.exp new file mode 100644 index 000000000000..930fda3a95c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-5.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-5.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-5.nix:2:3 + + … while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict + + error: expected a Boolean but found an integer: 15 diff --git a/tests/functional/lang/eval-fail-derivationStrict-5.nix b/tests/functional/lang/eval-fail-derivationStrict-5.nix new file mode 100644 index 000000000000..e74a8caf45c4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-5.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + __structuredAttrs = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-6.err.exp b/tests/functional/lang/eval-fail-derivationStrict-6.err.exp new file mode 100644 index 000000000000..323db3b37949 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-6.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-6.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-6.nix:2:3 + + … while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict + + error: expected a Boolean but found an integer: 15 diff --git a/tests/functional/lang/eval-fail-derivationStrict-6.nix b/tests/functional/lang/eval-fail-derivationStrict-6.nix new file mode 100644 index 000000000000..feda92b00d13 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-6.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + __ignoreNulls = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-7.err.exp b/tests/functional/lang/eval-fail-derivationStrict-7.err.exp new file mode 100644 index 000000000000..78ddd1573982 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-7.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-7.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-7.nix:2:3 + + … while evaluating attribute 'outputHashMode' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-7.nix:4:3: + 3| builder = 1; + 4| outputHashMode = 15; + | ^ + 5| } + + error: invalid value '15' for 'outputHashMode' attribute diff --git a/tests/functional/lang/eval-fail-derivationStrict-7.nix b/tests/functional/lang/eval-fail-derivationStrict-7.nix new file mode 100644 index 000000000000..1ffdb6fdbe99 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-7.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + outputHashMode = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-8.err.exp b/tests/functional/lang/eval-fail-derivationStrict-8.err.exp new file mode 100644 index 000000000000..ba4e0109c62d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-8.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-8.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-8.nix:2:3 + + … while evaluating attribute 'outputHashMode' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-8.nix:4:3: + 3| builder = 1; + 4| outputHashMode = "custom"; + | ^ + 5| } + + error: invalid value 'custom' for 'outputHashMode' attribute diff --git a/tests/functional/lang/eval-fail-derivationStrict-8.nix b/tests/functional/lang/eval-fail-derivationStrict-8.nix new file mode 100644 index 000000000000..38243416fdda --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-8.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + outputHashMode = "custom"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-9.err.exp b/tests/functional/lang/eval-fail-derivationStrict-9.err.exp new file mode 100644 index 000000000000..6bfbbc65b867 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-9.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-9.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-9.nix:2:3 + + … while evaluating attribute 'system' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-9.nix:4:3: + 3| builder = 1; + 4| system = { }; + | ^ + 5| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-9.nix b/tests/functional/lang/eval-fail-derivationStrict-9.nix new file mode 100644 index 000000000000..a3a567a14ce3 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-9.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = { }; +} diff --git a/tests/functional/lang/eval-fail-div-1.err.exp b/tests/functional/lang/eval-fail-div-1.err.exp new file mode 100644 index 000000000000..286e1f79c3c5 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-1.nix:1:1: + 1| builtins.div "foo" 1 + | ^ + 2| + + … while evaluating the first operand of the division + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-div-1.nix b/tests/functional/lang/eval-fail-div-1.nix new file mode 100644 index 000000000000..c6edce369753 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-1.nix @@ -0,0 +1 @@ +builtins.div "foo" 1 diff --git a/tests/functional/lang/eval-fail-div-2.err.exp b/tests/functional/lang/eval-fail-div-2.err.exp new file mode 100644 index 000000000000..20e6e710c096 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-2.nix:1:1: + 1| builtins.div 1 "foo" + | ^ + 2| + + … while evaluating the second operand of the division + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-div-2.nix b/tests/functional/lang/eval-fail-div-2.nix new file mode 100644 index 000000000000..71d9947719eb --- /dev/null +++ b/tests/functional/lang/eval-fail-div-2.nix @@ -0,0 +1 @@ +builtins.div 1 "foo" diff --git a/tests/functional/lang/eval-fail-div-3.err.exp b/tests/functional/lang/eval-fail-div-3.err.exp new file mode 100644 index 000000000000..6d263c956ae4 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-3.nix:1:1: + 1| builtins.div "foo" 0 + | ^ + 2| + + error: division by zero diff --git a/tests/functional/lang/eval-fail-div-3.nix b/tests/functional/lang/eval-fail-div-3.nix new file mode 100644 index 000000000000..11c606baac7c --- /dev/null +++ b/tests/functional/lang/eval-fail-div-3.nix @@ -0,0 +1 @@ +builtins.div "foo" 0 diff --git a/tests/functional/lang/eval-fail-elem-1.err.exp b/tests/functional/lang/eval-fail-elem-1.err.exp new file mode 100644 index 000000000000..cd0a2281414a --- /dev/null +++ b/tests/functional/lang/eval-fail-elem-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'elem' builtin + at /pwd/lang/eval-fail-elem-1.nix:1:1: + 1| builtins.elem 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.elem + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-elem-1.nix b/tests/functional/lang/eval-fail-elem-1.nix new file mode 100644 index 000000000000..eecc965cb438 --- /dev/null +++ b/tests/functional/lang/eval-fail-elem-1.nix @@ -0,0 +1 @@ +builtins.elem 1 "foo" diff --git a/tests/functional/lang/eval-fail-elemAt-1.err.exp b/tests/functional/lang/eval-fail-elemAt-1.err.exp new file mode 100644 index 000000000000..3ea5b0aef0b5 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-1.nix:1:1: + 1| builtins.elemAt "foo" (-1) + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.elemAt' + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-elemAt-1.nix b/tests/functional/lang/eval-fail-elemAt-1.nix new file mode 100644 index 000000000000..27de601ebac2 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-1.nix @@ -0,0 +1 @@ +builtins.elemAt "foo" (-1) diff --git a/tests/functional/lang/eval-fail-elemAt-2.err.exp b/tests/functional/lang/eval-fail-elemAt-2.err.exp new file mode 100644 index 000000000000..6d746c7739e5 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-2.nix:1:1: + 1| builtins.elemAt [ ] (-1) + | ^ + 2| + + error: 'builtins.elemAt' called with index -1 on a list of size 0 diff --git a/tests/functional/lang/eval-fail-elemAt-2.nix b/tests/functional/lang/eval-fail-elemAt-2.nix new file mode 100644 index 000000000000..cf48f089f2fe --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-2.nix @@ -0,0 +1 @@ +builtins.elemAt [ ] (-1) diff --git a/tests/functional/lang/eval-fail-elemAt-3.err.exp b/tests/functional/lang/eval-fail-elemAt-3.err.exp new file mode 100644 index 000000000000..1f34afb45871 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-3.nix:1:1: + 1| builtins.elemAt [ "foo" ] 3 + | ^ + 2| + + error: 'builtins.elemAt' called with index 3 on a list of size 1 diff --git a/tests/functional/lang/eval-fail-elemAt-3.nix b/tests/functional/lang/eval-fail-elemAt-3.nix new file mode 100644 index 000000000000..8493015def6e --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-3.nix @@ -0,0 +1 @@ +builtins.elemAt [ "foo" ] 3 diff --git a/tests/functional/lang/eval-fail-filter-1.err.exp b/tests/functional/lang/eval-fail-filter-1.err.exp new file mode 100644 index 000000000000..d3c55d9f651e --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-1.nix:1:1: + 1| builtins.filter 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.filter + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-filter-1.nix b/tests/functional/lang/eval-fail-filter-1.nix new file mode 100644 index 000000000000..37b379f87d50 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-1.nix @@ -0,0 +1 @@ +builtins.filter 1 "foo" diff --git a/tests/functional/lang/eval-fail-filter-2.err.exp b/tests/functional/lang/eval-fail-filter-2.err.exp new file mode 100644 index 000000000000..5e5d0ecc5a79 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-2.nix:1:1: + 1| builtins.filter 1 [ "foo" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.filter + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-filter-2.nix b/tests/functional/lang/eval-fail-filter-2.nix new file mode 100644 index 000000000000..177adfa82c1d --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-2.nix @@ -0,0 +1 @@ +builtins.filter 1 [ "foo" ] diff --git a/tests/functional/lang/eval-fail-filter-3.err.exp b/tests/functional/lang/eval-fail-filter-3.err.exp new file mode 100644 index 000000000000..ba5a87e27567 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-3.nix:1:1: + 1| builtins.filter (_: 5) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the filtering function passed to builtins.filter + + error: expected a Boolean but found an integer: 5 diff --git a/tests/functional/lang/eval-fail-filter-3.nix b/tests/functional/lang/eval-fail-filter-3.nix new file mode 100644 index 000000000000..c262f25bf6c5 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-3.nix @@ -0,0 +1 @@ +builtins.filter (_: 5) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-filterSource-1.err.exp b/tests/functional/lang/eval-fail-filterSource-1.err.exp new file mode 100644 index 000000000000..cf2d153f0deb --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-1.nix:1:1: + 1| builtins.filterSource [ ] [ ] + | ^ + 2| + + … while evaluating the second argument (the path to filter) passed to 'builtins.filterSource' + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-1.nix b/tests/functional/lang/eval-fail-filterSource-1.nix new file mode 100644 index 000000000000..5833e0de595e --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-1.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-2.err.exp b/tests/functional/lang/eval-fail-filterSource-2.err.exp new file mode 100644 index 000000000000..e3302f032f9d --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-2.nix:1:1: + 1| builtins.filterSource [ ] "foo" + | ^ + 2| + + … while evaluating the second argument (the path to filter) passed to 'builtins.filterSource' + + error: string 'foo' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-filterSource-2.nix b/tests/functional/lang/eval-fail-filterSource-2.nix new file mode 100644 index 000000000000..5fc0b68cb0d5 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-2.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] "foo" diff --git a/tests/functional/lang/eval-fail-filterSource-3.err.exp b/tests/functional/lang/eval-fail-filterSource-3.err.exp new file mode 100644 index 000000000000..5750f8198c63 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-3.nix:1:1: + 1| builtins.filterSource [ ] ./. + | ^ + 2| + + … while evaluating the first argument passed to builtins.filterSource + + error: expected a function but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-3.nix b/tests/functional/lang/eval-fail-filterSource-3.nix new file mode 100644 index 000000000000..6c4124aadb38 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-3.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] ./. diff --git a/tests/functional/lang/eval-fail-filterSource-4.err.exp b/tests/functional/lang/eval-fail-filterSource-4.err.exp new file mode 100644 index 000000000000..4318616ee8fb --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-4.nix:1:1: + 1| builtins.filterSource (_: 1) ./. + | ^ + 2| + + … while adding path '/pwd/lang' + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-filterSource-4.nix b/tests/functional/lang/eval-fail-filterSource-4.nix new file mode 100644 index 000000000000..28a729c614b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-4.nix @@ -0,0 +1 @@ +builtins.filterSource (_: 1) ./. diff --git a/tests/functional/lang/eval-fail-filterSource-5.err.exp b/tests/functional/lang/eval-fail-filterSource-5.err.exp new file mode 100644 index 000000000000..fe017b2efb8d --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-5.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-5.nix:1:1: + 1| builtins.filterSource (_: _: 1) ./. + | ^ + 2| + + … while adding path '/pwd/lang' + + … while evaluating the return value of the path filter function + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-filterSource-5.nix b/tests/functional/lang/eval-fail-filterSource-5.nix new file mode 100644 index 000000000000..dbb0decee0ee --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-5.nix @@ -0,0 +1 @@ +builtins.filterSource (_: _: 1) ./. diff --git a/tests/functional/lang/eval-fail-floor-1.err.exp b/tests/functional/lang/eval-fail-floor-1.err.exp new file mode 100644 index 000000000000..2e5fb0aae3db --- /dev/null +++ b/tests/functional/lang/eval-fail-floor-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'floor' builtin + at /pwd/lang/eval-fail-floor-1.nix:1:1: + 1| builtins.floor "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.floor + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-floor-1.nix b/tests/functional/lang/eval-fail-floor-1.nix new file mode 100644 index 000000000000..1d4e1dcf5b7c --- /dev/null +++ b/tests/functional/lang/eval-fail-floor-1.nix @@ -0,0 +1 @@ +builtins.floor "foo" diff --git a/tests/functional/lang/eval-fail-foldlPrime-1.err.exp b/tests/functional/lang/eval-fail-foldlPrime-1.err.exp new file mode 100644 index 000000000000..45540098ea4d --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-1.nix:1:1: + 1| builtins.foldl' 1 "foo" true + | ^ + 2| + + … while evaluating the first argument passed to builtins.foldlStrict + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-foldlPrime-1.nix b/tests/functional/lang/eval-fail-foldlPrime-1.nix new file mode 100644 index 000000000000..e7790d4191a6 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-1.nix @@ -0,0 +1 @@ +builtins.foldl' 1 "foo" true diff --git a/tests/functional/lang/eval-fail-foldlPrime-2.err.exp b/tests/functional/lang/eval-fail-foldlPrime-2.err.exp new file mode 100644 index 000000000000..7597793eb28b --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-2.nix:1:1: + 1| builtins.foldl' (_: 1) "foo" true + | ^ + 2| + + … while evaluating the third argument passed to builtins.foldlStrict + + error: expected a list but found a Boolean: true diff --git a/tests/functional/lang/eval-fail-foldlPrime-2.nix b/tests/functional/lang/eval-fail-foldlPrime-2.nix new file mode 100644 index 000000000000..9599f33cd22b --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-2.nix @@ -0,0 +1 @@ +builtins.foldl' (_: 1) "foo" true diff --git a/tests/functional/lang/eval-fail-foldlPrime-3.err.exp b/tests/functional/lang/eval-fail-foldlPrime-3.err.exp new file mode 100644 index 000000000000..8ba0a3dfb819 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-3.nix:1:1: + 1| builtins.foldl' (_: 1) "foo" [ true ] + | ^ + 2| + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-foldlPrime-3.nix b/tests/functional/lang/eval-fail-foldlPrime-3.nix new file mode 100644 index 000000000000..6e570d9c70b1 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-3.nix @@ -0,0 +1 @@ +builtins.foldl' (_: 1) "foo" [ true ] diff --git a/tests/functional/lang/eval-fail-foldlPrime-4.err.exp b/tests/functional/lang/eval-fail-foldlPrime-4.err.exp new file mode 100644 index 000000000000..9ead5b3929e0 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-4.err.exp @@ -0,0 +1,24 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:1: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:21: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + … in the left operand of the AND (&&) operator + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:26: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + error: expected a Boolean but found a string: "foo" + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:26: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-foldlPrime-4.nix b/tests/functional/lang/eval-fail-foldlPrime-4.nix new file mode 100644 index 000000000000..c34df35dd147 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-4.nix @@ -0,0 +1 @@ +builtins.foldl' (a: b: a && b) "foo" [ true ] diff --git a/tests/functional/lang/eval-fail-functionArgs-1.err.exp b/tests/functional/lang/eval-fail-functionArgs-1.err.exp new file mode 100644 index 000000000000..4f15c0655d92 --- /dev/null +++ b/tests/functional/lang/eval-fail-functionArgs-1.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'functionArgs' builtin + at /pwd/lang/eval-fail-functionArgs-1.nix:1:1: + 1| builtins.functionArgs { } + | ^ + 2| + + error: 'functionArgs' requires a function diff --git a/tests/functional/lang/eval-fail-functionArgs-1.nix b/tests/functional/lang/eval-fail-functionArgs-1.nix new file mode 100644 index 000000000000..151e9fa976d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-functionArgs-1.nix @@ -0,0 +1 @@ +builtins.functionArgs { } diff --git a/tests/functional/lang/eval-fail-genList-1.err.exp b/tests/functional/lang/eval-fail-genList-1.err.exp new file mode 100644 index 000000000000..deea7e7ead79 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-1.nix:1:1: + 1| builtins.genList 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.genList + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-genList-1.nix b/tests/functional/lang/eval-fail-genList-1.nix new file mode 100644 index 000000000000..276147c1d369 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-1.nix @@ -0,0 +1 @@ +builtins.genList 1 "foo" diff --git a/tests/functional/lang/eval-fail-genList-2.err.exp b/tests/functional/lang/eval-fail-genList-2.err.exp new file mode 100644 index 000000000000..dbbab7d525d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-2.nix:1:1: + 1| builtins.genList 1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.genList + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-genList-2.nix b/tests/functional/lang/eval-fail-genList-2.nix new file mode 100644 index 000000000000..0f5bb6a3b668 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-2.nix @@ -0,0 +1 @@ +builtins.genList 1 2 diff --git a/tests/functional/lang/eval-fail-genList-3.err.exp b/tests/functional/lang/eval-fail-genList-3.err.exp new file mode 100644 index 000000000000..62878d88eb5c --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-3.err.exp @@ -0,0 +1,20 @@ +error: + … while evaluating list element at index 0 + + … from call site + at /pwd/lang/eval-fail-genList-3.nix:1:19: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-genList-3.nix:1:19: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| + + error: cannot add a string to an integer + at /pwd/lang/eval-fail-genList-3.nix:1:26: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-genList-3.nix b/tests/functional/lang/eval-fail-genList-3.nix new file mode 100644 index 000000000000..82b8b30bc1a2 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-3.nix @@ -0,0 +1 @@ +builtins.genList (x: x + "foo") 2 diff --git a/tests/functional/lang/eval-fail-genList-4.err.exp b/tests/functional/lang/eval-fail-genList-4.err.exp new file mode 100644 index 000000000000..a60ffe6d775b --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-4.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-4.nix:1:1: + 1| builtins.genList false (-3) + | ^ + 2| + + error: cannot create list of size -3 diff --git a/tests/functional/lang/eval-fail-genList-4.nix b/tests/functional/lang/eval-fail-genList-4.nix new file mode 100644 index 000000000000..b4f4ee6188a1 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-4.nix @@ -0,0 +1 @@ +builtins.genList false (-3) diff --git a/tests/functional/lang/eval-fail-getAttr-1.err.exp b/tests/functional/lang/eval-fail-getAttr-1.err.exp new file mode 100644 index 000000000000..d2ff63a710ad --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-1.nix:1:1: + 1| builtins.getAttr [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.getAttr + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-1.nix b/tests/functional/lang/eval-fail-getAttr-1.nix new file mode 100644 index 000000000000..80b92f45dec9 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-1.nix @@ -0,0 +1 @@ +builtins.getAttr [ ] [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-2.err.exp b/tests/functional/lang/eval-fail-getAttr-2.err.exp new file mode 100644 index 000000000000..e4a8afe799ed --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-2.nix:1:1: + 1| builtins.getAttr "foo" [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.getAttr + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-2.nix b/tests/functional/lang/eval-fail-getAttr-2.nix new file mode 100644 index 000000000000..6d836c033255 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-2.nix @@ -0,0 +1 @@ +builtins.getAttr "foo" [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-3.err.exp b/tests/functional/lang/eval-fail-getAttr-3.err.exp new file mode 100644 index 000000000000..e3a3dab0589b --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-3.nix:1:1: + 1| builtins.getAttr "foo" { } + | ^ + 2| + + … in the attribute set under consideration + + error: attribute 'foo' missing diff --git a/tests/functional/lang/eval-fail-getAttr-3.nix b/tests/functional/lang/eval-fail-getAttr-3.nix new file mode 100644 index 000000000000..248a616bf33d --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-3.nix @@ -0,0 +1 @@ +builtins.getAttr "foo" { } diff --git a/tests/functional/lang/eval-fail-getEnv-1.err.exp b/tests/functional/lang/eval-fail-getEnv-1.err.exp new file mode 100644 index 000000000000..92db4e5e4270 --- /dev/null +++ b/tests/functional/lang/eval-fail-getEnv-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getEnv' builtin + at /pwd/lang/eval-fail-getEnv-1.nix:1:1: + 1| builtins.getEnv [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.getEnv + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getEnv-1.nix b/tests/functional/lang/eval-fail-getEnv-1.nix new file mode 100644 index 000000000000..5e7cb3ea1b02 --- /dev/null +++ b/tests/functional/lang/eval-fail-getEnv-1.nix @@ -0,0 +1 @@ +builtins.getEnv [ ] diff --git a/tests/functional/lang/eval-fail-groupBy-1.err.exp b/tests/functional/lang/eval-fail-groupBy-1.err.exp new file mode 100644 index 000000000000..02d0415b2bd9 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-1.nix:1:1: + 1| builtins.groupBy 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.groupBy + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-groupBy-1.nix b/tests/functional/lang/eval-fail-groupBy-1.nix new file mode 100644 index 000000000000..8fb41ada1352 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-1.nix @@ -0,0 +1 @@ +builtins.groupBy 1 "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-2.err.exp b/tests/functional/lang/eval-fail-groupBy-2.err.exp new file mode 100644 index 000000000000..4a9b7b6daad1 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-2.nix:1:1: + 1| builtins.groupBy (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.groupBy + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-2.nix b/tests/functional/lang/eval-fail-groupBy-2.nix new file mode 100644 index 000000000000..fe91a7bae88d --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-2.nix @@ -0,0 +1 @@ +builtins.groupBy (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-3.err.exp b/tests/functional/lang/eval-fail-groupBy-3.err.exp new file mode 100644 index 000000000000..4adc418338ed --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-3.nix:1:1: + 1| builtins.groupBy (x: x) [ + | ^ + 2| "foo" + + … while evaluating the return value of the grouping function passed to builtins.groupBy + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-groupBy-3.nix b/tests/functional/lang/eval-fail-groupBy-3.nix new file mode 100644 index 000000000000..a4165783d99e --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-3.nix @@ -0,0 +1,5 @@ +builtins.groupBy (x: x) [ + "foo" + "bar" + 1 +] diff --git a/tests/functional/lang/eval-fail-hasAttr-1.err.exp b/tests/functional/lang/eval-fail-hasAttr-1.err.exp new file mode 100644 index 000000000000..b44ae1c0c77d --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hasAttr' builtin + at /pwd/lang/eval-fail-hasAttr-1.nix:1:1: + 1| builtins.hasAttr [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.hasAttr + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-1.nix b/tests/functional/lang/eval-fail-hasAttr-1.nix new file mode 100644 index 000000000000..8bdfbaaa7112 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-1.nix @@ -0,0 +1 @@ +builtins.hasAttr [ ] [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-2.err.exp b/tests/functional/lang/eval-fail-hasAttr-2.err.exp new file mode 100644 index 000000000000..cd52370acf4b --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hasAttr' builtin + at /pwd/lang/eval-fail-hasAttr-2.nix:1:1: + 1| builtins.hasAttr "foo" [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.hasAttr + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-2.nix b/tests/functional/lang/eval-fail-hasAttr-2.nix new file mode 100644 index 000000000000..d49909c76389 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-2.nix @@ -0,0 +1 @@ +builtins.hasAttr "foo" [ ] diff --git a/tests/functional/lang/eval-fail-hashString-1.err.exp b/tests/functional/lang/eval-fail-hashString-1.err.exp new file mode 100644 index 000000000000..ecce1a5d846f --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-1.nix:1:1: + 1| builtins.hashString 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.hashString + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-hashString-1.nix b/tests/functional/lang/eval-fail-hashString-1.nix new file mode 100644 index 000000000000..bbca23d43f7f --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-1.nix @@ -0,0 +1 @@ +builtins.hashString 1 { } diff --git a/tests/functional/lang/eval-fail-hashString-2.err.exp b/tests/functional/lang/eval-fail-hashString-2.err.exp new file mode 100644 index 000000000000..edbf5e23ff78 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-2.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-2.nix:1:1: + 1| builtins.hashString "foo" "content" + | ^ + 2| + + error: unknown hash algorithm 'foo', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512' +Try 'nix-instantiate --help' for more information. diff --git a/tests/functional/lang/eval-fail-hashString-2.nix b/tests/functional/lang/eval-fail-hashString-2.nix new file mode 100644 index 000000000000..9439b8b41d55 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-2.nix @@ -0,0 +1 @@ +builtins.hashString "foo" "content" diff --git a/tests/functional/lang/eval-fail-hashString-3.err.exp b/tests/functional/lang/eval-fail-hashString-3.err.exp new file mode 100644 index 000000000000..2126b0062572 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-3.nix:1:1: + 1| builtins.hashString "sha256" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.hashString + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-hashString-3.nix b/tests/functional/lang/eval-fail-hashString-3.nix new file mode 100644 index 000000000000..50b960c8d38c --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-3.nix @@ -0,0 +1 @@ +builtins.hashString "sha256" { } diff --git a/tests/functional/lang/eval-fail-head-1.err.exp b/tests/functional/lang/eval-fail-head-1.err.exp new file mode 100644 index 000000000000..3fcbbe29b68c --- /dev/null +++ b/tests/functional/lang/eval-fail-head-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'head' builtin + at /pwd/lang/eval-fail-head-1.nix:1:1: + 1| builtins.head 1 + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.head' + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-head-1.nix b/tests/functional/lang/eval-fail-head-1.nix new file mode 100644 index 000000000000..11aa7248706e --- /dev/null +++ b/tests/functional/lang/eval-fail-head-1.nix @@ -0,0 +1 @@ +builtins.head 1 diff --git a/tests/functional/lang/eval-fail-head-2.err.exp b/tests/functional/lang/eval-fail-head-2.err.exp new file mode 100644 index 000000000000..68cecc8fa1d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-head-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'head' builtin + at /pwd/lang/eval-fail-head-2.nix:1:1: + 1| builtins.head [ ] + | ^ + 2| + + error: 'builtins.head' called on an empty list diff --git a/tests/functional/lang/eval-fail-head-2.nix b/tests/functional/lang/eval-fail-head-2.nix new file mode 100644 index 000000000000..c7a77da4e20e --- /dev/null +++ b/tests/functional/lang/eval-fail-head-2.nix @@ -0,0 +1 @@ +builtins.head [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp b/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp new file mode 100644 index 000000000000..098084a3f4f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'intersectAttrs' builtin + at /pwd/lang/eval-fail-intersectAttrs-1.nix:1:1: + 1| builtins.intersectAttrs [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.intersectAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-1.nix b/tests/functional/lang/eval-fail-intersectAttrs-1.nix new file mode 100644 index 000000000000..5a8c133163d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-1.nix @@ -0,0 +1 @@ +builtins.intersectAttrs [ ] [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp b/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp new file mode 100644 index 000000000000..663df4f6147e --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'intersectAttrs' builtin + at /pwd/lang/eval-fail-intersectAttrs-2.nix:1:1: + 1| builtins.intersectAttrs { } [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.intersectAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-2.nix b/tests/functional/lang/eval-fail-intersectAttrs-2.nix new file mode 100644 index 000000000000..413ce5d58b3a --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-2.nix @@ -0,0 +1 @@ +builtins.intersectAttrs { } [ ] diff --git a/tests/functional/lang/eval-fail-length-1.err.exp b/tests/functional/lang/eval-fail-length-1.err.exp new file mode 100644 index 000000000000..7161a2fbf26f --- /dev/null +++ b/tests/functional/lang/eval-fail-length-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-length-1.nix:1:1: + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-length-1.nix b/tests/functional/lang/eval-fail-length-1.nix new file mode 100644 index 000000000000..972eb72c7697 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-1.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-length-2.err.exp b/tests/functional/lang/eval-fail-length-2.err.exp new file mode 100644 index 000000000000..8bd66a9a701a --- /dev/null +++ b/tests/functional/lang/eval-fail-length-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-length-2.nix:1:1: + 1| builtins.length "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-length-2.nix b/tests/functional/lang/eval-fail-length-2.nix new file mode 100644 index 000000000000..a42e1034ab07 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-2.nix @@ -0,0 +1 @@ +builtins.length "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-1.err.exp b/tests/functional/lang/eval-fail-lessThan-1.err.exp new file mode 100644 index 000000000000..09ec5d8ffdbe --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-1.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-1.nix:1:1: + 1| builtins.lessThan 1 "foo" + | ^ + 2| + + error: cannot compare an integer with a string; values are 1 and "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-1.nix b/tests/functional/lang/eval-fail-lessThan-1.nix new file mode 100644 index 000000000000..10cd489674ad --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-1.nix @@ -0,0 +1 @@ +builtins.lessThan 1 "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-2.err.exp b/tests/functional/lang/eval-fail-lessThan-2.err.exp new file mode 100644 index 000000000000..6a8a183fa74e --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-2.nix:1:1: + 1| builtins.lessThan { } { } + | ^ + 2| + + error: cannot compare a set with a set; values of that type are incomparable (values are { } and { }) diff --git a/tests/functional/lang/eval-fail-lessThan-2.nix b/tests/functional/lang/eval-fail-lessThan-2.nix new file mode 100644 index 000000000000..a20eca6bba14 --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-2.nix @@ -0,0 +1 @@ +builtins.lessThan { } { } diff --git a/tests/functional/lang/eval-fail-lessThan-3.err.exp b/tests/functional/lang/eval-fail-lessThan-3.err.exp new file mode 100644 index 000000000000..4a66625c00cd --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-3.nix:1:1: + 1| builtins.lessThan [ 1 2 ] [ "foo" ] + | ^ + 2| + + … while comparing two list elements + + error: cannot compare an integer with a string; values are 1 and "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-3.nix b/tests/functional/lang/eval-fail-lessThan-3.nix new file mode 100644 index 000000000000..bdea0fef172c --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-3.nix @@ -0,0 +1 @@ +builtins.lessThan [ 1 2 ] [ "foo" ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-1.err.exp b/tests/functional/lang/eval-fail-listToAttrs-1.err.exp new file mode 100644 index 000000000000..ce2ef38c8b52 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-1.nix:1:1: + 1| builtins.listToAttrs 1 + | ^ + 2| + + … while evaluating the argument passed to builtins.listToAttrs + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-1.nix b/tests/functional/lang/eval-fail-listToAttrs-1.nix new file mode 100644 index 000000000000..260f136e51ca --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-1.nix @@ -0,0 +1 @@ +builtins.listToAttrs 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-2.err.exp b/tests/functional/lang/eval-fail-listToAttrs-2.err.exp new file mode 100644 index 000000000000..9809d216a253 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-2.nix:1:1: + 1| builtins.listToAttrs [ 1 ] + | ^ + 2| + + … while evaluating an element of the list passed to builtins.listToAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-2.nix b/tests/functional/lang/eval-fail-listToAttrs-2.nix new file mode 100644 index 000000000000..2db54c3c7b52 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-2.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ 1 ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-3.err.exp b/tests/functional/lang/eval-fail-listToAttrs-3.err.exp new file mode 100644 index 000000000000..5c4886c1745f --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-3.nix:1:1: + 1| builtins.listToAttrs [ { } ] + | ^ + 2| + + … in a {name=...; value=...;} pair + + error: attribute 'name' missing diff --git a/tests/functional/lang/eval-fail-listToAttrs-3.nix b/tests/functional/lang/eval-fail-listToAttrs-3.nix new file mode 100644 index 000000000000..ce5b11798153 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-3.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { } ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-4.err.exp b/tests/functional/lang/eval-fail-listToAttrs-4.err.exp new file mode 100644 index 000000000000..cb284ad2cd38 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-4.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:1: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| + + … while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:26: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| + + error: expected a string but found an integer: 1 + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:26: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-listToAttrs-4.nix b/tests/functional/lang/eval-fail-listToAttrs-4.nix new file mode 100644 index 000000000000..a4bc7fd11c67 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-4.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { name = 1; } ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-5.err.exp b/tests/functional/lang/eval-fail-listToAttrs-5.err.exp new file mode 100644 index 000000000000..a03834b49a37 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-5.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-5.nix:1:1: + 1| builtins.listToAttrs [ { name = "foo"; } ] + | ^ + 2| + + … in a {name=...; value=...;} pair + + error: attribute 'value' missing diff --git a/tests/functional/lang/eval-fail-listToAttrs-5.nix b/tests/functional/lang/eval-fail-listToAttrs-5.nix new file mode 100644 index 000000000000..7cca5d0c7258 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-5.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { name = "foo"; } ] diff --git a/tests/functional/lang/eval-fail-map-1.err.exp b/tests/functional/lang/eval-fail-map-1.err.exp new file mode 100644 index 000000000000..65c0d5d3abdf --- /dev/null +++ b/tests/functional/lang/eval-fail-map-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'map' builtin + at /pwd/lang/eval-fail-map-1.nix:1:1: + 1| builtins.map 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.map + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-map-1.nix b/tests/functional/lang/eval-fail-map-1.nix new file mode 100644 index 000000000000..65f5cf8a67b2 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-1.nix @@ -0,0 +1 @@ +builtins.map 1 "foo" diff --git a/tests/functional/lang/eval-fail-map-2.err.exp b/tests/functional/lang/eval-fail-map-2.err.exp new file mode 100644 index 000000000000..cf9a3f636753 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'map' builtin + at /pwd/lang/eval-fail-map-2.nix:1:1: + 1| builtins.map 1 [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.map + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-map-2.nix b/tests/functional/lang/eval-fail-map-2.nix new file mode 100644 index 000000000000..16100c0a7905 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-2.nix @@ -0,0 +1 @@ +builtins.map 1 [ 1 ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-1.err.exp b/tests/functional/lang/eval-fail-mapAttrs-1.err.exp new file mode 100644 index 000000000000..9a10bb92f841 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mapAttrs' builtin + at /pwd/lang/eval-fail-mapAttrs-1.nix:1:1: + 1| builtins.mapAttrs [ ] [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.mapAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-1.nix b/tests/functional/lang/eval-fail-mapAttrs-1.nix new file mode 100644 index 000000000000..c913e838d81a --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-1.nix @@ -0,0 +1 @@ +builtins.mapAttrs [ ] [ ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-2.err.exp b/tests/functional/lang/eval-fail-mapAttrs-2.err.exp new file mode 100644 index 000000000000..9cc2b5495a58 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-2.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but a string: "" diff --git a/tests/functional/lang/eval-fail-mapAttrs-2.nix b/tests/functional/lang/eval-fail-mapAttrs-2.nix new file mode 100644 index 000000000000..b58ed8c3a660 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-2.nix @@ -0,0 +1 @@ +builtins.mapAttrs "" { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-mapAttrs-3.err.exp b/tests/functional/lang/eval-fail-mapAttrs-3.err.exp new file mode 100644 index 000000000000..f4c48ee9a48f --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-3.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but a string: "foo1" diff --git a/tests/functional/lang/eval-fail-mapAttrs-3.nix b/tests/functional/lang/eval-fail-mapAttrs-3.nix new file mode 100644 index 000000000000..6fc4de8251cd --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-3.nix @@ -0,0 +1 @@ +builtins.mapAttrs (x: x + "1") { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-mapAttrs-4.err.exp b/tests/functional/lang/eval-fail-mapAttrs-4.err.exp new file mode 100644 index 000000000000..bd18ceeac946 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-4.err.exp @@ -0,0 +1,16 @@ +error: + … while evaluating the attribute 'foo' + + … while calling anonymous lambda + at /pwd/lang/eval-fail-mapAttrs-4.nix:1:23: + 1| builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } + | ^ + 2| + + … while evaluating a path segment + at /pwd/lang/eval-fail-mapAttrs-4.nix:1:30: + 1| builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } + | ^ + 2| + + error: cannot coerce an integer to a string: 1 diff --git a/tests/functional/lang/eval-fail-mapAttrs-4.nix b/tests/functional/lang/eval-fail-mapAttrs-4.nix new file mode 100644 index 000000000000..2ad09b602434 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-4.nix @@ -0,0 +1 @@ +builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-match-1.err.exp b/tests/functional/lang/eval-fail-match-1.err.exp new file mode 100644 index 000000000000..2f757a6834c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-1.nix:1:1: + 1| builtins.match 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.match + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-match-1.nix b/tests/functional/lang/eval-fail-match-1.nix new file mode 100644 index 000000000000..3eb26c017c53 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-1.nix @@ -0,0 +1 @@ +builtins.match 1 { } diff --git a/tests/functional/lang/eval-fail-match-2.err.exp b/tests/functional/lang/eval-fail-match-2.err.exp new file mode 100644 index 000000000000..e415f5ccd3e1 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-2.nix:1:1: + 1| builtins.match "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.match + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-match-2.nix b/tests/functional/lang/eval-fail-match-2.nix new file mode 100644 index 000000000000..35eb16cfbf2d --- /dev/null +++ b/tests/functional/lang/eval-fail-match-2.nix @@ -0,0 +1 @@ +builtins.match "foo" { } diff --git a/tests/functional/lang/eval-fail-match-3.err.exp b/tests/functional/lang/eval-fail-match-3.err.exp new file mode 100644 index 000000000000..d69577bd7163 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-3.nix:1:1: + 1| builtins.match "(.*" "" + | ^ + 2| + + error: invalid regular expression '(.*' diff --git a/tests/functional/lang/eval-fail-match-3.nix b/tests/functional/lang/eval-fail-match-3.nix new file mode 100644 index 000000000000..8115807f9220 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-3.nix @@ -0,0 +1 @@ +builtins.match "(.*" "" diff --git a/tests/functional/lang/eval-fail-mul-1.err.exp b/tests/functional/lang/eval-fail-mul-1.err.exp new file mode 100644 index 000000000000..c03284987c9f --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-mul-1.nix:1:1: + 1| builtins.mul "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the multiplication + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-mul-1.nix b/tests/functional/lang/eval-fail-mul-1.nix new file mode 100644 index 000000000000..a16333824671 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-1.nix @@ -0,0 +1 @@ +builtins.mul "foo" 1 diff --git a/tests/functional/lang/eval-fail-mul-2.err.exp b/tests/functional/lang/eval-fail-mul-2.err.exp new file mode 100644 index 000000000000..16744abe7310 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-mul-2.nix:1:1: + 1| builtins.mul 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the multiplication + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-mul-2.nix b/tests/functional/lang/eval-fail-mul-2.nix new file mode 100644 index 000000000000..57e2ef6925f0 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-2.nix @@ -0,0 +1 @@ +builtins.mul 1 "foo" diff --git a/tests/functional/lang/eval-fail-parseDrvName-1.err.exp b/tests/functional/lang/eval-fail-parseDrvName-1.err.exp new file mode 100644 index 000000000000..e5fbc2b6ec84 --- /dev/null +++ b/tests/functional/lang/eval-fail-parseDrvName-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'parseDrvName' builtin + at /pwd/lang/eval-fail-parseDrvName-1.nix:1:1: + 1| builtins.parseDrvName 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.parseDrvName + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-parseDrvName-1.nix b/tests/functional/lang/eval-fail-parseDrvName-1.nix new file mode 100644 index 000000000000..f524d500f5fc --- /dev/null +++ b/tests/functional/lang/eval-fail-parseDrvName-1.nix @@ -0,0 +1 @@ +builtins.parseDrvName 1 diff --git a/tests/functional/lang/eval-fail-partition-1.err.exp b/tests/functional/lang/eval-fail-partition-1.err.exp new file mode 100644 index 000000000000..d989969bd552 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-1.nix:1:1: + 1| builtins.partition 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.partition + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-partition-1.nix b/tests/functional/lang/eval-fail-partition-1.nix new file mode 100644 index 000000000000..0452d11101d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-1.nix @@ -0,0 +1 @@ +builtins.partition 1 "foo" diff --git a/tests/functional/lang/eval-fail-partition-2.err.exp b/tests/functional/lang/eval-fail-partition-2.err.exp new file mode 100644 index 000000000000..e3fef6848fcf --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-2.nix:1:1: + 1| builtins.partition (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.partition + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-partition-2.nix b/tests/functional/lang/eval-fail-partition-2.nix new file mode 100644 index 000000000000..1a7219c5f3d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-2.nix @@ -0,0 +1 @@ +builtins.partition (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-partition-3.err.exp b/tests/functional/lang/eval-fail-partition-3.err.exp new file mode 100644 index 000000000000..a1257c7a6b1c --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-3.nix:1:1: + 1| builtins.partition (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the partition function passed to builtins.partition + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-partition-3.nix b/tests/functional/lang/eval-fail-partition-3.nix new file mode 100644 index 000000000000..07aa480fc18d --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-3.nix @@ -0,0 +1 @@ +builtins.partition (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-pathExists-1.err.exp b/tests/functional/lang/eval-fail-pathExists-1.err.exp new file mode 100644 index 000000000000..3349247966fd --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'pathExists' builtin + at /pwd/lang/eval-fail-pathExists-1.nix:1:1: + 1| builtins.pathExists [ ] + | ^ + 2| + + … while realising the context of a path + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-pathExists-1.nix b/tests/functional/lang/eval-fail-pathExists-1.nix new file mode 100644 index 000000000000..83bda9afca95 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-1.nix @@ -0,0 +1 @@ +builtins.pathExists [ ] diff --git a/tests/functional/lang/eval-fail-pathExists-2.err.exp b/tests/functional/lang/eval-fail-pathExists-2.err.exp new file mode 100644 index 000000000000..184e47238d82 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'pathExists' builtin + at /pwd/lang/eval-fail-pathExists-2.nix:1:1: + 1| builtins.pathExists "zorglub" + | ^ + 2| + + … while realising the context of a path + + error: string 'zorglub' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-pathExists-2.nix b/tests/functional/lang/eval-fail-pathExists-2.nix new file mode 100644 index 000000000000..da594ca602b5 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-2.nix @@ -0,0 +1 @@ +builtins.pathExists "zorglub" diff --git a/tests/functional/lang/eval-fail-placeholder-1.err.exp b/tests/functional/lang/eval-fail-placeholder-1.err.exp new file mode 100644 index 000000000000..436095448054 --- /dev/null +++ b/tests/functional/lang/eval-fail-placeholder-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'placeholder' builtin + at /pwd/lang/eval-fail-placeholder-1.nix:1:1: + 1| builtins.placeholder [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.placeholder + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-placeholder-1.nix b/tests/functional/lang/eval-fail-placeholder-1.nix new file mode 100644 index 000000000000..ecc934a5e357 --- /dev/null +++ b/tests/functional/lang/eval-fail-placeholder-1.nix @@ -0,0 +1 @@ +builtins.placeholder [ ] diff --git a/tests/functional/lang/eval-fail-removeAttrs-1.err.exp b/tests/functional/lang/eval-fail-removeAttrs-1.err.exp new file mode 100644 index 000000000000..ddfde967f164 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-1.nix:1:1: + 1| builtins.removeAttrs "" "" + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-1.nix b/tests/functional/lang/eval-fail-removeAttrs-1.nix new file mode 100644 index 000000000000..83fe63003c9d --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-1.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-2.err.exp b/tests/functional/lang/eval-fail-removeAttrs-2.err.exp new file mode 100644 index 000000000000..ab668aaea0e4 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-2.nix:1:1: + 1| builtins.removeAttrs "" [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-2.nix b/tests/functional/lang/eval-fail-removeAttrs-2.nix new file mode 100644 index 000000000000..ecedf26cf975 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-2.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" [ 1 ] diff --git a/tests/functional/lang/eval-fail-removeAttrs-3.err.exp b/tests/functional/lang/eval-fail-removeAttrs-3.err.exp new file mode 100644 index 000000000000..1937cd1be86d --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-3.nix:1:1: + 1| builtins.removeAttrs "" [ "1" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-3.nix b/tests/functional/lang/eval-fail-removeAttrs-3.nix new file mode 100644 index 000000000000..4bc9e02394b7 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-3.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" [ "1" ] diff --git a/tests/functional/lang/eval-fail-replaceStrings-1.err.exp b/tests/functional/lang/eval-fail-replaceStrings-1.err.exp new file mode 100644 index 000000000000..b0bc7c758c73 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-1.nix:1:1: + 1| builtins.replaceStrings 0 0 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.replaceStrings + + error: expected a list but found an integer: 0 diff --git a/tests/functional/lang/eval-fail-replaceStrings-1.nix b/tests/functional/lang/eval-fail-replaceStrings-1.nix new file mode 100644 index 000000000000..7e9edc28d26f --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-1.nix @@ -0,0 +1 @@ +builtins.replaceStrings 0 0 { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-2.err.exp b/tests/functional/lang/eval-fail-replaceStrings-2.err.exp new file mode 100644 index 000000000000..ee289eded6f1 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-2.nix:1:1: + 1| builtins.replaceStrings [ ] 0 { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.replaceStrings + + error: expected a list but found an integer: 0 diff --git a/tests/functional/lang/eval-fail-replaceStrings-2.nix b/tests/functional/lang/eval-fail-replaceStrings-2.nix new file mode 100644 index 000000000000..57311720207f --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-2.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ ] 0 { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-3.err.exp b/tests/functional/lang/eval-fail-replaceStrings-3.err.exp new file mode 100644 index 000000000000..3e71cfa75c9a --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-3.nix:1:1: + 1| builtins.replaceStrings [ 0 ] [ ] { } + | ^ + 2| + + error: 'from' and 'to' arguments passed to builtins.replaceStrings have different lengths diff --git a/tests/functional/lang/eval-fail-replaceStrings-3.nix b/tests/functional/lang/eval-fail-replaceStrings-3.nix new file mode 100644 index 000000000000..c7cdac4a9665 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-3.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ 0 ] [ ] { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-4.err.exp b/tests/functional/lang/eval-fail-replaceStrings-4.err.exp new file mode 100644 index 000000000000..f4e834090531 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-4.nix:1:1: + 1| builtins.replaceStrings [ 1 ] [ "new" ] { } + | ^ + 2| + + … while evaluating one of the strings to replace passed to builtins.replaceStrings + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-replaceStrings-4.nix b/tests/functional/lang/eval-fail-replaceStrings-4.nix new file mode 100644 index 000000000000..cc700ea54a50 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-4.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ 1 ] [ "new" ] { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-5.err.exp b/tests/functional/lang/eval-fail-replaceStrings-5.err.exp new file mode 100644 index 000000000000..75e2b5e8b345 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-5.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-5.nix:1:1: + 1| builtins.replaceStrings [ "oo" ] [ true ] "foo" + | ^ + 2| + + … while evaluating one of the replacement strings passed to builtins.replaceStrings + + error: expected a string but found a Boolean: true diff --git a/tests/functional/lang/eval-fail-replaceStrings-5.nix b/tests/functional/lang/eval-fail-replaceStrings-5.nix new file mode 100644 index 000000000000..40010d845e40 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-5.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ "oo" ] [ true ] "foo" diff --git a/tests/functional/lang/eval-fail-replaceStrings-6.err.exp b/tests/functional/lang/eval-fail-replaceStrings-6.err.exp new file mode 100644 index 000000000000..5ced1affaf21 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-6.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-6.nix:1:1: + 1| builtins.replaceStrings [ "old" ] [ "new" ] { } + | ^ + 2| + + … while evaluating the third argument passed to builtins.replaceStrings + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-6.nix b/tests/functional/lang/eval-fail-replaceStrings-6.nix new file mode 100644 index 000000000000..2953b4888bef --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-6.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ "old" ] [ "new" ] { } diff --git a/tests/functional/lang/eval-fail-sort-1.err.exp b/tests/functional/lang/eval-fail-sort-1.err.exp new file mode 100644 index 000000000000..2797d344d6aa --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-1.nix:1:1: + 1| builtins.sort 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.sort + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sort-1.nix b/tests/functional/lang/eval-fail-sort-1.nix new file mode 100644 index 000000000000..12d3a102872d --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-1.nix @@ -0,0 +1 @@ +builtins.sort 1 "foo" diff --git a/tests/functional/lang/eval-fail-sort-2.err.exp b/tests/functional/lang/eval-fail-sort-2.err.exp new file mode 100644 index 000000000000..462ade0e447b --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-2.nix:1:1: + 1| builtins.sort 1 [ "foo" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.sort + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-2.nix b/tests/functional/lang/eval-fail-sort-2.nix new file mode 100644 index 000000000000..27ffb1d3633c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-2.nix @@ -0,0 +1 @@ +builtins.sort 1 [ "foo" ] diff --git a/tests/functional/lang/eval-fail-sort-3.err.exp b/tests/functional/lang/eval-fail-sort-3.err.exp new file mode 100644 index 000000000000..0d86a7f4d1a3 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-3.nix:1:1: + 1| builtins.sort (_: 1) [ + | ^ + 2| "foo" + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-3.nix b/tests/functional/lang/eval-fail-sort-3.nix new file mode 100644 index 000000000000..1fcd434d07f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-3.nix @@ -0,0 +1,4 @@ +builtins.sort (_: 1) [ + "foo" + "bar" +] diff --git a/tests/functional/lang/eval-fail-sort-4.err.exp b/tests/functional/lang/eval-fail-sort-4.err.exp new file mode 100644 index 000000000000..517eefeb5c98 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-4.nix:1:1: + 1| builtins.sort (_: _: 1) [ + | ^ + 2| "foo" + + … while evaluating the return value of the sorting function passed to builtins.sort + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-4.nix b/tests/functional/lang/eval-fail-sort-4.nix new file mode 100644 index 000000000000..89868de9abb7 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-4.nix @@ -0,0 +1,4 @@ +builtins.sort (_: _: 1) [ + "foo" + "bar" +] diff --git a/tests/functional/lang/eval-fail-sort-5.err.exp b/tests/functional/lang/eval-fail-sort-5.err.exp new file mode 100644 index 000000000000..f4c66dbd466c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-5.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-5.nix:1:1: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … while calling anonymous lambda + at /pwd/lang/eval-fail-sort-5.nix:1:19: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … in the argument of the not operator + at /pwd/lang/eval-fail-sort-5.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-sort-5.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + error: cannot compare a string with a set; values are "foo" and { } diff --git a/tests/functional/lang/eval-fail-sort-5.nix b/tests/functional/lang/eval-fail-sort-5.nix new file mode 100644 index 000000000000..bd50e0b325da --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-5.nix @@ -0,0 +1,4 @@ +builtins.sort (a: b: a <= b) [ + "foo" + { } +] diff --git a/tests/functional/lang/eval-fail-sort-6.err.exp b/tests/functional/lang/eval-fail-sort-6.err.exp new file mode 100644 index 000000000000..302a27f2361b --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-6.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-6.nix:1:1: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … while calling anonymous lambda + at /pwd/lang/eval-fail-sort-6.nix:1:19: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … in the argument of the not operator + at /pwd/lang/eval-fail-sort-6.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-sort-6.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + error: cannot compare a set with a set; values of that type are incomparable (values are { } and { }) diff --git a/tests/functional/lang/eval-fail-sort-6.nix b/tests/functional/lang/eval-fail-sort-6.nix new file mode 100644 index 000000000000..3c527596d82c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-6.nix @@ -0,0 +1,4 @@ +builtins.sort (a: b: a <= b) [ + { } + { } +] diff --git a/tests/functional/lang/eval-fail-split-1.err.exp b/tests/functional/lang/eval-fail-split-1.err.exp new file mode 100644 index 000000000000..c8ac7085741c --- /dev/null +++ b/tests/functional/lang/eval-fail-split-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-1.nix:1:1: + 1| builtins.split 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.split + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-split-1.nix b/tests/functional/lang/eval-fail-split-1.nix new file mode 100644 index 000000000000..ad702a1a3a80 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-1.nix @@ -0,0 +1 @@ +builtins.split 1 { } diff --git a/tests/functional/lang/eval-fail-split-2.err.exp b/tests/functional/lang/eval-fail-split-2.err.exp new file mode 100644 index 000000000000..a912771d0639 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-2.nix:1:1: + 1| builtins.split "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.split + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-split-2.nix b/tests/functional/lang/eval-fail-split-2.nix new file mode 100644 index 000000000000..43489c5374e2 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-2.nix @@ -0,0 +1 @@ +builtins.split "foo" { } diff --git a/tests/functional/lang/eval-fail-split-3.err.exp b/tests/functional/lang/eval-fail-split-3.err.exp new file mode 100644 index 000000000000..a7cfd394188b --- /dev/null +++ b/tests/functional/lang/eval-fail-split-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-3.nix:1:1: + 1| builtins.split "f(o*o" "1foo2" + | ^ + 2| + + error: invalid regular expression 'f(o*o' diff --git a/tests/functional/lang/eval-fail-split-3.nix b/tests/functional/lang/eval-fail-split-3.nix new file mode 100644 index 000000000000..f45294e93050 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-3.nix @@ -0,0 +1 @@ +builtins.split "f(o*o" "1foo2" diff --git a/tests/functional/lang/eval-fail-splitVersion-1.err.exp b/tests/functional/lang/eval-fail-splitVersion-1.err.exp new file mode 100644 index 000000000000..f7eb7614d89d --- /dev/null +++ b/tests/functional/lang/eval-fail-splitVersion-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'splitVersion' builtin + at /pwd/lang/eval-fail-splitVersion-1.nix:1:1: + 1| builtins.splitVersion 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.splitVersion + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-splitVersion-1.nix b/tests/functional/lang/eval-fail-splitVersion-1.nix new file mode 100644 index 000000000000..43941d1b6640 --- /dev/null +++ b/tests/functional/lang/eval-fail-splitVersion-1.nix @@ -0,0 +1 @@ +builtins.splitVersion 1 diff --git a/tests/functional/lang/eval-fail-storePath-1.err.exp b/tests/functional/lang/eval-fail-storePath-1.err.exp new file mode 100644 index 000000000000..711ed13ed8ce --- /dev/null +++ b/tests/functional/lang/eval-fail-storePath-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'storePath' builtin + at /pwd/lang/eval-fail-storePath-1.nix:1:1: + 1| builtins.storePath true + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.storePath' + + error: cannot coerce a Boolean to a string: true diff --git a/tests/functional/lang/eval-fail-storePath-1.nix b/tests/functional/lang/eval-fail-storePath-1.nix new file mode 100644 index 000000000000..d4ce341022d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-storePath-1.nix @@ -0,0 +1 @@ +builtins.storePath true diff --git a/tests/functional/lang/eval-fail-stringLength-1.err.exp b/tests/functional/lang/eval-fail-stringLength-1.err.exp new file mode 100644 index 000000000000..37d63fd493c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-stringLength-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'stringLength' builtin + at /pwd/lang/eval-fail-stringLength-1.nix:1:1: + 1| builtins.stringLength { } + | ^ + 2| + + … while evaluating the argument passed to builtins.stringLength + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-stringLength-1.nix b/tests/functional/lang/eval-fail-stringLength-1.nix new file mode 100644 index 000000000000..0a5ba6d8a842 --- /dev/null +++ b/tests/functional/lang/eval-fail-stringLength-1.nix @@ -0,0 +1 @@ +builtins.stringLength { } diff --git a/tests/functional/lang/eval-fail-sub-1.err.exp b/tests/functional/lang/eval-fail-sub-1.err.exp new file mode 100644 index 000000000000..769bf252bbb8 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-sub-1.nix:1:1: + 1| builtins.sub "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the subtraction + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sub-1.nix b/tests/functional/lang/eval-fail-sub-1.nix new file mode 100644 index 000000000000..cbff4ca1ed72 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-1.nix @@ -0,0 +1 @@ +builtins.sub "foo" 1 diff --git a/tests/functional/lang/eval-fail-sub-2.err.exp b/tests/functional/lang/eval-fail-sub-2.err.exp new file mode 100644 index 000000000000..ee5ed4618a53 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-sub-2.nix:1:1: + 1| builtins.sub 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the subtraction + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sub-2.nix b/tests/functional/lang/eval-fail-sub-2.nix new file mode 100644 index 000000000000..4a3cfc6e0254 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-2.nix @@ -0,0 +1 @@ +builtins.sub 1 "foo" diff --git a/tests/functional/lang/eval-fail-substring-1.err.exp b/tests/functional/lang/eval-fail-substring-1.err.exp new file mode 100644 index 000000000000..84e61c17c075 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-1.nix:1:1: + 1| builtins.substring { } "foo" true + | ^ + 2| + + … while evaluating the first argument (the start offset) passed to builtins.substring + + error: expected an integer but found a set: { } diff --git a/tests/functional/lang/eval-fail-substring-1.nix b/tests/functional/lang/eval-fail-substring-1.nix new file mode 100644 index 000000000000..c12228550bf8 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-1.nix @@ -0,0 +1 @@ +builtins.substring { } "foo" true diff --git a/tests/functional/lang/eval-fail-substring-2.err.exp b/tests/functional/lang/eval-fail-substring-2.err.exp new file mode 100644 index 000000000000..641fde5a83dc --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-2.nix:1:1: + 1| builtins.substring 3 "foo" true + | ^ + 2| + + … while evaluating the second argument (the substring length) passed to builtins.substring + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-substring-2.nix b/tests/functional/lang/eval-fail-substring-2.nix new file mode 100644 index 000000000000..358b0d083506 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-2.nix @@ -0,0 +1 @@ +builtins.substring 3 "foo" true diff --git a/tests/functional/lang/eval-fail-substring-3.err.exp b/tests/functional/lang/eval-fail-substring-3.err.exp new file mode 100644 index 000000000000..67b12cab2709 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-3.nix:1:1: + 1| builtins.substring 0 3 { } + | ^ + 2| + + … while evaluating the third argument (the string) passed to builtins.substring + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-substring-3.nix b/tests/functional/lang/eval-fail-substring-3.nix new file mode 100644 index 000000000000..f52921e48103 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-3.nix @@ -0,0 +1 @@ +builtins.substring 0 3 { } diff --git a/tests/functional/lang/eval-fail-substring-4.err.exp b/tests/functional/lang/eval-fail-substring-4.err.exp new file mode 100644 index 000000000000..cdaa11b20227 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-4.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-4.nix:1:1: + 1| builtins.substring (-3) 3 "sometext" + | ^ + 2| + + error: negative start position in 'substring' diff --git a/tests/functional/lang/eval-fail-substring-4.nix b/tests/functional/lang/eval-fail-substring-4.nix new file mode 100644 index 000000000000..82813b2c5838 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-4.nix @@ -0,0 +1 @@ +builtins.substring (-3) 3 "sometext" diff --git a/tests/functional/lang/eval-fail-tail-1.err.exp b/tests/functional/lang/eval-fail-tail-1.err.exp new file mode 100644 index 000000000000..2473767896ff --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'tail' builtin + at /pwd/lang/eval-fail-tail-1.nix:1:1: + 1| builtins.tail 1 + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.tail' + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-tail-1.nix b/tests/functional/lang/eval-fail-tail-1.nix new file mode 100644 index 000000000000..229ab7c8d7a1 --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-1.nix @@ -0,0 +1 @@ +builtins.tail 1 diff --git a/tests/functional/lang/eval-fail-tail-2.err.exp b/tests/functional/lang/eval-fail-tail-2.err.exp new file mode 100644 index 000000000000..63abbf2cba3d --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'tail' builtin + at /pwd/lang/eval-fail-tail-2.nix:1:1: + 1| builtins.tail [ ] + | ^ + 2| + + error: 'builtins.tail' called on an empty list diff --git a/tests/functional/lang/eval-fail-tail-2.nix b/tests/functional/lang/eval-fail-tail-2.nix new file mode 100644 index 000000000000..3bd318d1c33f --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-2.nix @@ -0,0 +1 @@ +builtins.tail [ ] diff --git a/tests/functional/lang/eval-fail-toPath-1.err.exp b/tests/functional/lang/eval-fail-toPath-1.err.exp new file mode 100644 index 000000000000..6e977c6a2bab --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-toPath-1.nix:1:1: + 1| builtins.toPath [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-toPath-1.nix b/tests/functional/lang/eval-fail-toPath-1.nix new file mode 100644 index 000000000000..cd96c880b803 --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-1.nix @@ -0,0 +1 @@ +builtins.toPath [ ] diff --git a/tests/functional/lang/eval-fail-toPath-2.err.exp b/tests/functional/lang/eval-fail-toPath-2.err.exp new file mode 100644 index 000000000000..f7d421b76d8d --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-toPath-2.nix:1:1: + 1| builtins.toPath "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: string 'foo' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-toPath-2.nix b/tests/functional/lang/eval-fail-toPath-2.nix new file mode 100644 index 000000000000..4bcb7fc14778 --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-2.nix @@ -0,0 +1 @@ +builtins.toPath "foo" diff --git a/tests/functional/lang/eval-fail-toString-1.err.exp b/tests/functional/lang/eval-fail-toString-1.err.exp new file mode 100644 index 000000000000..d73b9ad84c50 --- /dev/null +++ b/tests/functional/lang/eval-fail-toString-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toString' builtin + at /pwd/lang/eval-fail-toString-1.nix:1:1: + 1| builtins.toString { a = 1; } + | ^ + 2| + + … while evaluating the first argument passed to builtins.toString + + error: cannot coerce a set to a string: { a = 1; } diff --git a/tests/functional/lang/eval-fail-toString-1.nix b/tests/functional/lang/eval-fail-toString-1.nix new file mode 100644 index 000000000000..e4b2d8d7955b --- /dev/null +++ b/tests/functional/lang/eval-fail-toString-1.nix @@ -0,0 +1 @@ +builtins.toString { a = 1; } diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp new file mode 100644 index 000000000000..7aced51b97d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'zipAttrsWith' builtin + at /pwd/lang/eval-fail-zipAttrsWith-1.nix:1:1: + 1| builtins.zipAttrsWith [ ] [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.zipAttrsWith + + error: expected a function but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-1.nix b/tests/functional/lang/eval-fail-zipAttrsWith-1.nix new file mode 100644 index 000000000000..1f44609cd2ae --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-1.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith [ ] [ 1 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp new file mode 100644 index 000000000000..0a27e2d0195e --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'zipAttrsWith' builtin + at /pwd/lang/eval-fail-zipAttrsWith-2.nix:1:1: + 1| builtins.zipAttrsWith (_: 1) [ 1 ] + | ^ + 2| + + … while evaluating a value of the list passed as second argument to builtins.zipAttrsWith + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-2.nix b/tests/functional/lang/eval-fail-zipAttrsWith-2.nix new file mode 100644 index 000000000000..331da91a535b --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-2.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith (_: 1) [ 1 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp new file mode 100644 index 000000000000..780643b9df28 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but an integer: 1 + at /pwd/lang/eval-fail-zipAttrsWith-3.nix:1:24: + 1| builtins.zipAttrsWith (_: 1) [ { foo = 1; } ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-3.nix b/tests/functional/lang/eval-fail-zipAttrsWith-3.nix new file mode 100644 index 000000000000..6ac25c011010 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-3.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith (_: 1) [ { foo = 1; } ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp new file mode 100644 index 000000000000..034d84254198 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the attribute 'foo' + + … from call site + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:24: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + … while calling anonymous lambda + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:27: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + … while evaluating a path segment + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:34: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + error: cannot coerce a list to a string: [ 1 2 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-4.nix b/tests/functional/lang/eval-fail-zipAttrsWith-4.nix new file mode 100644 index 000000000000..dbe63d0b564d --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-4.nix @@ -0,0 +1,4 @@ +builtins.zipAttrsWith (a: b: a + b) [ + { foo = 1; } + { foo = 2; } +] From 940825bce6d2496e58e6f892125ad02038267ccd Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Thu, 30 Apr 2026 12:34:46 -0400 Subject: [PATCH 078/364] Filter systemd socket activation sockets by name When available, use the `LISTEN_FDNAMES` environment variable provided by systemd to listen only to the correct sockets in `serveUnixSocket`. Filtering makes it possible to differentiate different activation sockets connecting to the same daemon process. --- src/libcmd/include/nix/cmd/unix-socket-server.hh | 7 +++++++ src/libcmd/unix/unix-socket-server.cc | 8 ++++++++ src/nix/unix/daemon.cc | 1 + src/nix/unix/store-roots-daemon.cc | 1 + 4 files changed, 17 insertions(+) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa79317..544b988e2d95 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -55,6 +55,13 @@ struct ServeUnixSocketOptions mode_t socketMode = 0666; #ifndef _WIN32 + /** + * Name of the socket for socket activation, as included in `LISTEN_FDNAMES` + * Ordinarily the name of the socket unit, e.g. `nix-daemon.socket` + * If this field is empty, no name filtering will be performed. + */ + std::string activationName = ""; + /** * Additional file descriptor to poll. Useful for doing a self-pipe trick * https://cr.yp.to/docs/selfpipe.html. diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba462207..1794148b6ff9 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -5,6 +5,7 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/util/signals.hh" +#include "nix/util/strings.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" @@ -65,9 +66,16 @@ PeerInfo getPeerInfo(Descriptor remote) if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid())) throw Error("unexpected systemd environment variables"); + + auto fdNames = tokenizeString>(getEnv("LISTEN_FDNAMES").value_or(""), ":"); auto count = string2Int(*listenFds); assert(count); for (unsigned int i = 0; i < count; ++i) { + // Not all implementations of LISTEN_FDS will implement names, + // listen anyway if we do not have enough names + if (i < fdNames.size() && options.activationName != "" && fdNames[i] != options.activationName) + continue; + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); closeOnExec(fdSocket.get()); listeningSockets.push_back(std::move(fdSocket)); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 05e47f79c36b..c50c9f594228 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -285,6 +285,7 @@ static void daemonLoop( { .socketPath = std::move(socketPath), .socketMode = 0666, + .activationName = "nix-daemon.socket", .auxiliaryFd = sigChldPipe.pipe.readSide.get(), .onAuxiliaryFdPollin = []() { diff --git a/src/nix/unix/store-roots-daemon.cc b/src/nix/unix/store-roots-daemon.cc index 85b0a67a9156..b803c8cea6cf 100644 --- a/src/nix/unix/store-roots-daemon.cc +++ b/src/nix/unix/store-roots-daemon.cc @@ -44,6 +44,7 @@ struct CmdRootsDaemon : StoreConfigCommand { .socketPath = gcSocketPath, .socketMode = 0666, + .activationName = "nix-roots-daemon.socket", }, [&](AutoCloseFD remote, std::function closeListeners) { std::thread([&, remote = std::move(remote)]() mutable { From a79b0f4dc1521d27ea5c9829ff422914843775ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2026 20:06:48 +0200 Subject: [PATCH 079/364] LocalStore::addToStore(): Handle negative path info cache entry If we have a negative path info cache entry for the path being added (e.g. due to a prior call to maybeQueryPathInfo()), then we need to use isValidPathUncached(), otherwise addToStore() will fail. --- src/libstore/local-store.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 53d7456f94f6..0bc7b6e1b6c8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1013,7 +1013,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ - pathInfoCache->lock()->erase(path); + invalidatePathInfoCacheFor(path); } const PublicKeys & LocalStore::getPublicKeys() @@ -1049,17 +1049,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF auto realPath = toRealPath(info.path); /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ + from a build hook (whose parent process already acquired a + lock on this path). */ if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); - if (repair || !isValidPath(info.path)) { + /* The path may have been created by another process in the meantime, so check again. */ + if (repair || !isValidPathUncached(info.path)) { deletePath(realPath); /* While restoring the path from the NAR, compute the hash - of the NAR. */ + of the NAR. */ HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource{source, hashSink}; @@ -1129,7 +1130,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF } registerValidPath(info); - } + } else + // We may have a negative cache entry for this path, so get rid of it. + invalidatePathInfoCacheFor(info.path); outputLock.setDeletion(true); } @@ -1246,7 +1249,8 @@ StorePath LocalStore::addToStoreFromDump( PathLocks outputLock({realPath}); - if (repair || !isValidPath(dstPath)) { + /* The path may have been created by another process in the meantime, so check again. */ + if (repair || !isValidPathUncached(dstPath)) { deletePath(realPath); @@ -1293,7 +1297,9 @@ StorePath LocalStore::addToStoreFromDump( auto info = ValidPathInfo::makeFromCA(*this, name, std::move(desc), narHash.hash); info.narSize = narHash.numBytesDigested; registerValidPath(info); - } + } else + // We may have a negative cache entry for this path, so get rid of it. + invalidatePathInfoCacheFor(dstPath); outputLock.setDeletion(true); } From 026e930912d459290fac3eabbf11b6eb31ce681d Mon Sep 17 00:00:00 2001 From: ryota2357 Date: Sat, 18 Oct 2025 11:55:40 +0900 Subject: [PATCH 080/364] nix-profile{,-daemon}.fish: set NIX_PROFILES to use $NIX_LINK --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile.fish.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 1a20dffd2459..93cb3c45a55b 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -53,7 +53,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index abf716cec6fc..201a56438950 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -58,7 +58,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" From 0072b3271f6f23429805a6f7ea878a9f1b682cb6 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Fri, 1 May 2026 14:16:00 -0400 Subject: [PATCH 081/364] Ignore keep-{outputs,derivations} for delete `keep-outputs` and `keep-derivations` were not meant to affect delete operations. This change makes them only read when using the `WholeStore` variant type. Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/delete-keep.md | 8 +++ src/libstore/gc.cc | 65 ++++++++++--------- .../include/nix/store/local-settings.hh | 6 ++ tests/functional/delete-no-keep.sh | 32 +++++++++ tests/functional/meson.build | 1 + 5 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 doc/manual/rl-next/delete-keep.md create mode 100755 tests/functional/delete-no-keep.sh diff --git a/doc/manual/rl-next/delete-keep.md b/doc/manual/rl-next/delete-keep.md new file mode 100644 index 000000000000..0332e0e3933e --- /dev/null +++ b/doc/manual/rl-next/delete-keep.md @@ -0,0 +1,8 @@ +--- +synopsis: "Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands" +prs: [15776] +--- + +Setting `keep-derivations = true` and trying to delete a derivation with realised outputs would previously fail. +Same with `keep-outputs = true` and trying to delete an output that still has derivers. +These options no longer affect the deletion commands, and are now documented as such. diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 701fc66e69d0..3b9e8dbf5cd2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -357,8 +357,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) const auto & gcSettings = config->getLocalSettings().getGCSettings(); bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool keepOutputs = gcSettings.keepOutputs; - bool keepDerivations = gcSettings.keepDerivations; boost::unordered_flat_set> roots, dead, alive; @@ -382,16 +380,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::condition_variable wakeup; - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively, even if they aren't in the - pathsToDelete). So disable them. */ - if (std::holds_alternative(options.pathsToDelete) && options.ignoreLiveness) { - keepOutputs = false; - keepDerivations = false; - } - if (shouldDelete) deletePath(reservedPath); @@ -625,12 +613,23 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alive.insert(start); try { StorePathSet closure; + bool includeOutputs = false; + bool includeDerivers = false; + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + includeOutputs = gcSettings.keepOutputs; + includeDerivers = gcSettings.keepDerivations; + }, + [](const StorePathSet &) {}, + }, + options.pathsToDelete); computeFSClosure( *path, closure, /* flipDirection */ false, - keepOutputs, - keepDerivations); + includeOutputs, + includeDerivers); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { @@ -675,22 +674,28 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a derivation, then we only want to delete this derivation if - * we can also delete all its outputs, so visit the derivation outputs. */ - if (keepDerivations && path->isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) - if (maybeOutPath && isValidPath(*maybeOutPath) - && queryPathInfo(*maybeOutPath)->deriver == *path) - enqueue(*maybeOutPath); - } - - /* If keep-outputs is set, we only want to delete this path if we - * can also delete its derivers, so visit the derivers. */ - if (keepOutputs) { - auto derivers = queryValidDerivers(*path); - for (auto & i : derivers) - enqueue(i); - } + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + /* If keep-derivations is set and this is a derivation, then we only want to delete this + * derivation if we can also delete all its outputs, so visit the derivation outputs. */ + if (gcSettings.keepDerivations && path->isDerivation()) + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && isValidPath(*maybeOutPath) + && queryPathInfo(*maybeOutPath)->deriver == path) + enqueue(*maybeOutPath); + + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ + if (gcSettings.keepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + }, + [](const StorePathSet &) {}, + }, + options.pathsToDelete); } } for (auto & path : topoSortPaths(visited)) { diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 4fe28818d2b8..7381b5b8e766 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -61,6 +61,9 @@ struct GCSettings : public virtual Config collector still deletes store paths that are used only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to `true`. + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-outputs"}, }; @@ -80,6 +83,9 @@ struct GCSettings : public virtual Config store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if `keep-outputs` is also turned on). + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-derivations"}, }; diff --git a/tests/functional/delete-no-keep.sh b/tests/functional/delete-no-keep.sh new file mode 100755 index 000000000000..8ac64fd44493 --- /dev/null +++ b/tests/functional/delete-no-keep.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +deleteNoKeep() { + keep="$1" + + clearStore + drvPath=$(nix-instantiate simple.nix) + outPath=$(nix build -f simple.nix --no-link --print-out-paths) + + { + echo "keep-outputs = false" + echo "keep-derivations = false" + echo "keep-$keep = true" + } >> "$test_nix_conf" + + if [[ "$keep" = "outputs" ]]; then + nix store delete "$outPath" + [[ ! -e "$outPath" ]] || fail "$outPath should have been deleted" + else + nix store delete "$drvPath" + [[ ! -e "$drvPath" ]] || fail "$drvPath should have been deleted" + fi +} + +if isDaemonNewer "2.35pre"; then + deleteNoKeep outputs + deleteNoKeep derivations +fi diff --git a/tests/functional/meson.build b/tests/functional/meson.build index e2668c716ed8..ecb9ef87f663 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -179,6 +179,7 @@ suites = [ 'help.sh', 'symlinks.sh', 'external-builders.sh', + 'delete-no-keep.sh', ], 'workdir' : meson.current_source_dir(), }, From 6bf83e259a83d0324584977db81d0b66d9e649af Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:38:15 -0400 Subject: [PATCH 082/364] Add `--also-referrers` to `nix store delete` This makes collecting garbage within a closure more ergonomic. If you build `hello` from scratch, it'll also build `glibc` and `glibc-static`. `glibc-static` depends on `glibc`, but is not part of `hello`'s closure. This normally prevents deletion of `glibc`, but using this flag allows `glibc-static` to also be deleted as long as it is dead. Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/closure-gc.md | 7 ++-- src/libstore/daemon.cc | 12 ++++-- src/libstore/gc.cc | 37 +++++++++++------- src/libstore/include/nix/store/gc-store.hh | 12 +++++- .../include/nix/store/worker-protocol.hh | 8 +++- src/libstore/remote-store.cc | 12 ++++-- src/libstore/worker-protocol.cc | 22 +++++++++-- src/nix/nix-store/nix-store.cc | 4 +- src/nix/store-delete.cc | 12 +++++- tests/functional/dependencies2.nix | 39 +++++++++++++++++++ tests/functional/gc-closure.sh | 34 ++++++++++++---- 11 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 tests/functional/dependencies2.nix diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md index fc24d5e08b9b..945a78dc9f8c 100644 --- a/doc/manual/rl-next/closure-gc.md +++ b/doc/manual/rl-next/closure-gc.md @@ -1,9 +1,8 @@ --- synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" issues: 7239 -prs: 15236 +prs: [15236, 15727] --- -`nix store delete --recursive --skip-alive` can be used to collect garbage -within a closure, in which case it will only collect the dead paths that are -part of the closure of its arguments. +`nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. +The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 52e1c121c3e7..8a02ea6ae149 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -746,14 +746,17 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); } else { auto paths = WorkerProto::Serialise::read(*store, rconn); if (options.action != GCAction::gcDeleteSpecific && paths.empty()) options.pathsToDelete = GCOptions::WholeStore{}; else - options.pathsToDelete = paths; + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = paths, + .deleteReferrers = false, + }; } conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields @@ -761,8 +764,9 @@ static void performOp( readInt(conn.from); readInt(conn.from); - if (options.action == GCAction::gcDeleteDead && std::holds_alternative(options.pathsToDelete) - && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (options.action == GCAction::gcDeleteDead + && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { throw Error( "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3b9e8dbf5cd2..645c4c6f97b1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -361,8 +361,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) boost::unordered_flat_set> roots, dead, alive; /* Return early if nothing to delete */ - if (std::holds_alternative(options.pathsToDelete) - && std::get(options.pathsToDelete).empty()) + if (std::visit( + overloaded{ + [](const GCOptions::SpecificPaths & pathsToDelete) { return pathsToDelete.paths.empty(); }, + [](const GCOptions::WholeStore & _) { return false; }}, + options.pathsToDelete)) return; struct Shared @@ -621,7 +624,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) includeOutputs = gcSettings.keepOutputs; includeDerivers = gcSettings.keepDerivations; }, - [](const StorePathSet &) {}, + [](const GCOptions::SpecificPaths &) {}, }, options.pathsToDelete); computeFSClosure( @@ -642,14 +645,22 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) { - debug( - "cannot delete '%s' because '%s' is not in the specified paths to delete", - printStorePath(start), - printStorePath(*path)); + if (std::visit( + overloaded{ + [&](const GCOptions::SpecificPaths & pathsToDelete) { + if (!pathsToDelete.deleteReferrers && !pathsToDelete.paths.contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); + return true; + } + return false; + }, + [](const GCOptions::WholeStore & _) { return false; }, + }, + options.pathsToDelete)) return; - } { auto hashPart = path->hashPart(); @@ -693,7 +704,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) enqueue(i); } }, - [](const StorePathSet &) {}, + [](const GCOptions::SpecificPaths &) {}, }, options.pathsToDelete); } @@ -719,7 +730,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Either delete all garbage paths, or just the specified paths. */ std::visit( overloaded{ - [&](const StorePathSet & paths) { + [&](const GCOptions::SpecificPaths & pathsToDelete) { switch (options.action) { case GCOptions::gcDeleteDead: printInfo("deleting garbage within specified paths..."); @@ -732,7 +743,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); } - for (auto & i : paths) { + for (auto & i : pathsToDelete.paths) { maybeDeleteReferrersClosure(i); if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 015478e79fd3..0a0335016b74 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -42,6 +42,16 @@ struct GCOptions struct WholeStore {}; + struct SpecificPaths + { + StorePathSet paths; + + /** + * Allow dead referrers of candidate paths to also be deleted. + */ + bool deleteReferrers = false; + }; + GCAction action{gcDeleteDead}; /** @@ -55,7 +65,7 @@ struct GCOptions /** * The paths from which to delete. */ - using GCPaths = std::variant; + using GCPaths = std::variant; GCPaths pathsToDelete; /** diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index ea1cbb502d7a..64cab1494a76 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -126,9 +126,9 @@ struct WorkerProto static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; /** - * Feature for garbage collecting a specific set of paths. + * Feature for garbage collecting a specific set of paths and deleting referrers. */ - static constexpr std::string_view featureDeleteDeadSpecific = "delete-dead-specific"; + static constexpr std::string_view featureDeleteDeadSpecificReferrers = "delete-dead-specific-referrers"; /** * A unidirectional read connection, to be used by the read half of the @@ -345,6 +345,10 @@ template<> DECLARE_WORKER_SERIALISER(std::optional); template<> DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); + +template<> +DECLARE_WORKER_SERIALISER(GCOptions::SpecificPaths); + template<> DECLARE_WORKER_SERIALISER(GCOptions::GCPaths); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a2fb3a10d251..2c1ea0c63886 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -686,18 +686,23 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); - if (conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + bool supportsDeleteSpecificReferrers = + conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers); + + if (supportsDeleteSpecificReferrers) { conn->to << WorkerProto::Op::CollectGarbage; WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, options.pathsToDelete); } else { auto paths = std::visit( overloaded{ - [&](const StorePathSet & paths) { + [&](const GCOptions::SpecificPaths & paths) { if (options.action != GCOptions::gcDeleteSpecific) throw Error( "Your daemon version is too old to support garbage collecting a specific set of paths"); - return paths; + if (paths.deleteReferrers) + throw Error("Your daemon version is too old to support deleting referrers."); + return paths.paths; }, [](const GCOptions::WholeStore & _) { return StorePathSet{}; }, }, @@ -706,6 +711,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, paths); } + conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 31373a128f96..37aa4e31bec2 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -28,7 +28,7 @@ const WorkerProto::Version WorkerProto::latest = { std::string{ WorkerProto::featureRealisationWithPath, }, - std::string{WorkerProto::featureDeleteDeadSpecific}, + std::string{WorkerProto::featureDeleteDeadSpecificReferrers}, }, }; @@ -533,13 +533,29 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wr WorkerProto::write(store, conn, static_cast(info)); } +GCOptions::SpecificPaths +WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + GCOptions::SpecificPaths paths; + paths.paths = WorkerProto::Serialise::read(store, conn); + conn.from >> paths.deleteReferrers; + return paths; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const GCOptions::SpecificPaths & paths) +{ + WorkerProto::write(store, conn, paths.paths); + conn.to << paths.deleteReferrers; +} + GCOptions::GCPaths WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { uint8_t wholeStore; conn.from >> wholeStore; switch (wholeStore) { case 0: - return WorkerProto::Serialise::read(store, conn); + return WorkerProto::Serialise::read(store, conn); case 1: return GCOptions::WholeStore{}; default: @@ -552,7 +568,7 @@ void WorkerProto::Serialise::write( { std::visit( overloaded{ - [&](const StorePathSet paths) { + [&](const GCOptions::SpecificPaths paths) { conn.to << uint8_t{0}; WorkerProto::write(store, conn, paths); }, diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 15a4e878f5ac..63952ba63a17 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -727,7 +727,9 @@ static void opDelete(Strings opFlags, Strings opArgs) StorePathSet paths; for (auto & i : opArgs) paths.insert(store->followLinksToStorePath(i)); - options.pathsToDelete = std::move(paths); + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = std::move(paths), + }; auto & gcStore = require(*store); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index a1a387898491..c2f649ff568e 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -9,6 +9,7 @@ namespace nix { struct CmdStoreDelete : StorePathsCommand { GCOptions options{.action = GCOptions::gcDeleteSpecific}; + bool deleteReferrers = false; CmdStoreDelete() { @@ -25,6 +26,12 @@ struct CmdStoreDelete : StorePathsCommand "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", .handler = {&options.action, GCOptions::gcDeleteDead}, }); + + addFlag({ + .longName = "also-referrers", + .description = "Also allow deletion of any referrers of the specified paths.", + .handler = {&deleteReferrers, true}, + }); } std::string description() override @@ -46,7 +53,10 @@ struct CmdStoreDelete : StorePathsCommand StorePathSet paths; for (auto & path : storePaths) paths.insert(path); - options.pathsToDelete = std::move(paths); + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = std::move(paths), + .deleteReferrers = deleteReferrers, + }; GCResults results; Finally printer([&] { printFreed(false, results); }); diff --git a/tests/functional/dependencies2.nix b/tests/functional/dependencies2.nix new file mode 100644 index 000000000000..6300f0ac41fe --- /dev/null +++ b/tests/functional/dependencies2.nix @@ -0,0 +1,39 @@ +with import ./config.nix; + +let + + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + + input1 = mkDerivation { + name = "dependencies-input-1"; + buildCommand = "mkdir $out; echo FOO > $out/foo"; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + echo "$out" > $out2 + ''; + outputs = [ + "out" + "out2" + ]; + }; + +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + meta.description = "Random test package"; +} diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh index 03e2a0fc25d6..4b811e779717 100755 --- a/tests/functional/gc-closure.sh +++ b/tests/functional/gc-closure.sh @@ -5,25 +5,43 @@ source common.sh TODO_NixOS nix_gc_closure() { + ensureNoDeleteReferrer="${1}" + extraArg="${2:-""}" clearStore - nix build -f dependencies.nix input0_drv --out-link "$TEST_ROOT/gc-root" + nix build -f dependencies2.nix input0_drv --out-link "$TEST_ROOT/gc-root" input0=$(realpath "$TEST_ROOT/gc-root") - input1=$(nix build -f dependencies.nix input1_drv --no-link --print-out-paths) - input2=$(nix build -f dependencies.nix input2_drv --no-link --print-out-paths) - top=$(nix build -f dependencies.nix --no-link --print-out-paths) - somthing_else=$(nix store add-path ./dependencies.nix) + input1=$(nix build -f dependencies2.nix input1_drv --no-link --print-out-paths) + input2=$(nix build -f dependencies2.nix input2_drv --no-link --print-out-paths) + input2_out=$(printf "%s" "$input2" | head -n1) + input2_out2=$(printf "%s" "$input2" | tail -n1) + top=$(nix build -f dependencies2.nix --no-link --print-out-paths) + somthing_else=$(nix store add-path ./dependencies2.nix) if isDaemonNewer "2.35pre"; then + if [[ "$extraArg" != "--also-referrers" ]] && ! "$ensureNoDeleteReferrer"; then + nix store delete "$input2_out2" + fi # Check that nix store delete --recursive --skip-alive is best-effort (doesn't fail when some paths in the closure are alive) - nix store delete --recursive --skip-alive "$top" + # shellcheck disable=SC2086 # we want $extraArg to expand to nothing if unset + nix store delete --recursive --skip-alive $extraArg "$top" [[ ! -e "$top" ]] || fail "top should have been deleted" [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" - [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" [[ -e "$input1" ]] || fail "input1 is not in the closure of top, it shouldn't have been deleted" [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + if [[ "$extraArg" = "--also-referrers" ]]; then + [[ ! -e "$input2_out" ]] || fail "input2_out is part of top's closure and we can delete dead referrers, it should have been deleted" + elif "$ensureNoDeleteReferrer"; then + [[ -e "$input2_out" ]] || fail "input2_out is part of top's closure but we can't delete dead referrers, it shouldn't have been deleted" + else + [[ ! -e "$input2_out" ]] || fail "input2_out is not a gc root, is part of top's closure, and has no referrers, it should have been deleted" + fi + elif [[ "$extraArg" = "--also-referrers" ]]; then + expectStderr 1 nix store delete --recursive --also-referrers "$top" | grepQuiet "Your daemon version is too old to support deleting referrers" else expectStderr 1 nix store delete --recursive --skip-alive "$top" | grepQuiet "Your daemon version is too old to support garbage collecting a specific set of paths" fi } -nix_gc_closure +nix_gc_closure false +nix_gc_closure true +nix_gc_closure false --also-referrers From e6d05b4bab021a695efc0f26b706cf6df5ce182d Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:50:10 -0400 Subject: [PATCH 083/364] Add some examples for GCing a closure Signed-off-by: Lisanna Dettwyler --- src/nix/store-delete.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md index 026dccd0f19a..9b6fd9701d3e 100644 --- a/src/nix/store-delete.md +++ b/src/nix/store-delete.md @@ -8,6 +8,18 @@ R""( # nix store delete /nix/store/fdhrijyv3670djsgprx596nn89iwlj2s-hello-2.10 ``` +* Garbage collect a closure: + + ```console + # nix store delete --recursive --skip-alive nixpkgs#hello + ``` + +* Garbage collect a closure including dead referrers of the closure: + + ```console + # nix store delete --recursive --skip-alive --also-referrers nixpkgs#hello + ``` + # Description This command deletes the store paths specified by [*installables*](./nix.md#installables), From 66d9c62cde51b9c2e75f64c57a81b880467ef0fb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 03:16:09 +0300 Subject: [PATCH 084/364] Fix observe_string_cb to not assume a NUL terminated string With std::string_view::data() we really can't assume that the string ends up being NUL terminated. We already pass the length when needed, but we just don't use it in the test harness. All existing API consumers really should not be relying on this being the case in all situations, but this wasn't properly documented anywhere (though it is rather self-evident from the size parameter passed to the callback). --- src/libstore-tests/nix_api_store.cc | 4 ++-- src/libutil-c/nix_api_util.h | 1 + src/libutil-test-support/string_callback.cc | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 60626869a919..0162684daf4b 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -77,8 +77,8 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) { StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); ASSERT_NE(result, nullptr); - ASSERT_STREQ("name", result->path.name().data()); - ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); + ASSERT_EQ("name", result->path.name()); + ASSERT_EQ(PATH_SUFFIX.substr(1), result->path.to_string()); nix_store_path_free(result); } diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 66ea17522213..b48d9166d4be 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -163,6 +163,7 @@ typedef struct nix_c_context nix_c_context; * @brief Called to get the value of a string owned by Nix. * * The `start` data is borrowed and the function must not assume that the buffer persists after it returns. + * @warning Don't assume that the string is NUL-terminated. * * @param[in] start the string to copy. * @param[in] n the string length. diff --git a/src/libutil-test-support/string_callback.cc b/src/libutil-test-support/string_callback.cc index b64389e4adbd..70ff7ca7f4ef 100644 --- a/src/libutil-test-support/string_callback.cc +++ b/src/libutil-test-support/string_callback.cc @@ -5,7 +5,7 @@ namespace nix::testing { void observe_string_cb(const char * start, unsigned int n, void * user_data) { auto user_data_casted = reinterpret_cast(user_data); - *user_data_casted = std::string(start); + *user_data_casted = std::string(start, n); } } // namespace nix::testing From 5f90b0c5d5944638ead691ccc9437614ecce51f1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 03:53:20 +0300 Subject: [PATCH 085/364] Don't assume NUL terminated std::string_view in SQLiteStmt::Use::operator() std::string_view doesn't have to be NUL terminated - we were just getting lucky because everything that we passed in happened to be NUL terminated. This bug has existed since 7a9687ba30d579bc51e0aaf3193e0ab8d86400d2. --- src/libstore/sqlite.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index c65fc240c66f..4f39b61c5da7 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -177,7 +177,7 @@ SQLiteStmt::Use::~Use() SQLiteStmt::Use & SQLiteStmt::Use::operator()(std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), value.size(), SQLITE_TRANSIENT) != SQLITE_OK) SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); From 77ecdaf6d184e553492574d10c7648f3b9332779 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Sun, 26 Apr 2026 18:47:53 +0200 Subject: [PATCH 086/364] Add unit test to reproduce #15713. --- src/libstore-tests/meson.build | 1 + src/libstore-tests/outputs-query.cc | 114 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 src/libstore-tests/outputs-query.cc diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 78a1cc4a12dd..ab03bb2d2a0e 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -77,6 +77,7 @@ sources = files( 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', + 'outputs-query.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc new file mode 100644 index 000000000000..4be25e2ad3b8 --- /dev/null +++ b/src/libstore-tests/outputs-query.cc @@ -0,0 +1,114 @@ +// Regression tests for the functions in outputs-query.cc +// +// See https://github.com/NixOS/nix/issues/15713 + +#include + +#include "nix/store/outputs-query.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class OutputsQueryTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + EnableExperimentalFeature caFeature{"ca-derivations"}; + + ref store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openDummyStore(); + }(); + + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a simple floating CA derivation with a given name and no input + * derivations. + */ + Derivation makeLeafDrv(std::string name) + { + Derivation drv; + drv.name = std::move(name); + drv.platform = "x86_64-linux"; + drv.builder = "/bin/sh"; + drv.outputs = {{"out", caFloatingOutput()}}; + return drv; + } +}; + +/** + * Regression test for https://github.com/NixOS/nix/issues/15713 + * + * In a Fibonacci-style chain of floating CA derivations, the resolution + * algorithm used to call queryRealisation O(Fib(N)) times. + * This test verifies that memoization reduces this to O(N). + */ +TEST_F(OutputsQueryTest, fibonacciChainQueryCount) +{ + constexpr static size_t N = 10; + std::vector drvPaths; + + // d0, d1: leaf derivations + for (int i = 0; i < 2; ++i) { + drvPaths.push_back(store->writeDerivation(makeLeafDrv("d" + std::to_string(i)))); + } + + // d_i depends on d_{i-1} and d_{i-2} + for (size_t i = 2; i <= N; ++i) { + Derivation drv = makeLeafDrv("d" + std::to_string(i)); + drv.inputDrvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputDrvs.map[drvPaths[i - 2]].value.insert("out"); + drvPaths.push_back(store->writeDerivation(drv)); + } + + // Tracker for queryRealisation calls. + std::map callCounts; + std::map outPaths; + + QueryRealisationFun queryRealisation = [&](const DrvOutput & id) -> std::shared_ptr { + assert(id.outputName == "out"); + callCounts[id.drvPath]++; + + // Memoize mock output paths. + auto it = outPaths.find(id.drvPath); + if (it == outPaths.end()) { + auto hash = hashString(HashAlgorithm::SHA1, "mock-output-" + std::to_string(outPaths.size())); + it = outPaths.emplace(id.drvPath, StorePath(hash, "out")).first; + } + + return std::make_shared(UnkeyedRealisation{.outPath = it->second}); + }; + + auto result = deepQueryPartialDerivationOutput(*store, drvPaths[N], "out", nullptr, queryRealisation); + + ASSERT_TRUE(result); + + int totalCalls = 0; + for (auto & [path, count] : callCounts) { + totalCalls += count; + if (count > 1) + ADD_FAILURE() << "Derivation at " << store->printStorePath(path) << " was queried " << count + << " times (expected 1)"; + } + + // With full memoization (ResolveCache + RealisationCache), each derivation should be queried exactly once. + EXPECT_EQ(totalCalls, N + 1) << "queryRealisation called " << totalCalls << " times; expected exactly " << (N + 1); +} + +} // namespace nix From 6974f9e1257c9c2371ed3307fc4f5f748d2e5883 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Sat, 2 May 2026 13:42:19 -0400 Subject: [PATCH 087/364] Fix FreeBSD non-unity build Adds missing `` include. Signed-off-by: Lisanna Dettwyler --- src/libutil-tests/unix/unix-domain-socket.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil-tests/unix/unix-domain-socket.cc b/src/libutil-tests/unix/unix-domain-socket.cc index 439d3188ac2d..b5050e564585 100644 --- a/src/libutil-tests/unix/unix-domain-socket.cc +++ b/src/libutil-tests/unix/unix-domain-socket.cc @@ -6,6 +6,7 @@ #include #include +#include namespace nix { From 2c26a23a9a055df33850de3106488fa7e54d0c9d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 May 2026 13:58:44 -0400 Subject: [PATCH 088/364] Remove Perl bindings `nix-serve` is being deprecated, and so Hydra is the only active user of the perl bindings. As such, they have been moved to the hydra repo. Now that that is done, we can remove them from this repo. Delete `src/perl/` and all references to `nix-perl-bindings` across the build system, packaging, CI, and documentation. The `fetchers-substitute` NixOS test still uses `nix-serve`, which depends on Perl bindings, but now uses the older Nix from nixpkgs rather than this project's bindings. This is a stop-gap until we stop using `nix-serve` in that test (because it is deprecated) completely. --- ci/gha/tests/default.nix | 2 - doc/manual/source/development/debugging.md | 1 - flake.nix | 10 - meson.build | 5 - meson.options | 7 - .../clang-tidy/build_required_targets.py | 2 - .../common/clang-tidy/clean_compdb.py | 3 - packaging/components.nix | 2 - packaging/dev-shell.nix | 15 +- packaging/everything.nix | 20 +- packaging/hydra.nix | 12 - src/perl/.version | 1 - src/perl/.yath.rc.in | 2 - src/perl/MANIFEST | 7 - src/perl/lib/Nix/Config.pm.in | 24 - src/perl/lib/Nix/CopyClosure.pm | 61 --- src/perl/lib/Nix/Manifest.pm | 325 ------------- src/perl/lib/Nix/SSH.pm | 110 ----- src/perl/lib/Nix/Store.pm | 45 -- src/perl/lib/Nix/Store.xs | 430 ------------------ src/perl/lib/Nix/Utils.pm | 47 -- src/perl/lib/Nix/meson.build | 61 --- src/perl/meson.build | 196 -------- src/perl/meson.options | 30 -- src/perl/package.nix | 82 ---- src/perl/t/init.t | 13 - src/perl/t/meson.build | 15 - tests/nixos/fetchers-substitute.nix | 18 +- 28 files changed, 4 insertions(+), 1542 deletions(-) delete mode 120000 src/perl/.version delete mode 100644 src/perl/.yath.rc.in delete mode 100644 src/perl/MANIFEST delete mode 100644 src/perl/lib/Nix/Config.pm.in delete mode 100644 src/perl/lib/Nix/CopyClosure.pm delete mode 100644 src/perl/lib/Nix/Manifest.pm delete mode 100644 src/perl/lib/Nix/SSH.pm delete mode 100644 src/perl/lib/Nix/Store.pm delete mode 100644 src/perl/lib/Nix/Store.xs delete mode 100644 src/perl/lib/Nix/Utils.pm delete mode 100644 src/perl/lib/Nix/meson.build delete mode 100644 src/perl/meson.build delete mode 100644 src/perl/meson.options delete mode 100644 src/perl/package.nix delete mode 100644 src/perl/t/init.t delete mode 100644 src/perl/t/meson.build diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 5e1b23c7a36b..8d2383a92fb9 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -57,8 +57,6 @@ rec { nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 6578632d991a..35e4c71ec388 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -26,7 +26,6 @@ or GCC. This is useful when debugging memory corruption issues. ```console [nix-shell]$ export mesonBuildType=debugoptimized [nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm -[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl [nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" ``` diff --git a/flake.nix b/flake.nix index c9be5e1dd58c..8a2fe16c2770 100644 --- a/flake.nix +++ b/flake.nix @@ -328,12 +328,6 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } - // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { - # Some perl dependencies are broken on i686-linux. - # Since the support is only best-effort there, disable the perl - # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } # Add "passthru" tests // flatMapAttrs @@ -422,10 +416,6 @@ supportsCross = false; }; - "nix-perl-bindings" = { - supportsCross = false; - }; - "nix-clang-tidy-plugin" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index bef2e6221a53..b5d9434a1a84 100644 --- a/meson.build +++ b/meson.build @@ -47,11 +47,6 @@ subproject('libmain-c') asan_enabled = 'address' in get_option('b_sanitize') -# Language Bindings -if get_option('bindings') and not meson.is_cross_build() and not asan_enabled - subproject('perl') -endif - # Testing if get_option('unit-tests') subproject('libutil-test-support') diff --git a/meson.options b/meson.options index a306a84252ea..7b847beba831 100644 --- a/meson.options +++ b/meson.options @@ -14,13 +14,6 @@ option( description : 'Build unit tests', ) -option( - 'bindings', - type : 'boolean', - value : true, - description : 'Build language bindings (e.g. Perl)', -) - option( 'benchmarks', type : 'boolean', diff --git a/nix-meson-build-support/common/clang-tidy/build_required_targets.py b/nix-meson-build-support/common/clang-tidy/build_required_targets.py index d55acd74a21e..24e4f290c607 100755 --- a/nix-meson-build-support/common/clang-tidy/build_required_targets.py +++ b/nix-meson-build-support/common/clang-tidy/build_required_targets.py @@ -53,8 +53,6 @@ def main(): + [t for t in custom_commands if t.endswith(".gen.inc")] # Flex/Bison generated parsers + [t for t in custom_commands if t.endswith("-tab.cc")] - # Perl XS generated bindings - + [t for t in custom_commands if t.endswith(".cc") and "perl" in t.lower()] ) ninja_build(args.build_root, targets) diff --git a/nix-meson-build-support/common/clang-tidy/clean_compdb.py b/nix-meson-build-support/common/clang-tidy/clean_compdb.py index 659667209ac3..8087b0bf9b6e 100755 --- a/nix-meson-build-support/common/clang-tidy/clean_compdb.py +++ b/nix-meson-build-support/common/clang-tidy/clean_compdb.py @@ -48,9 +48,6 @@ def cmdfilter(item: dict) -> bool: # Filter out Flex/Bison generated parsers (generated code) if file.endswith("-tab.cc"): return False - # Filter out Perl XS generated bindings (generated code) - if "/perl/" in file and file.endswith(".cc"): - return False return True return [chomp(x) for x in compdb if cmdfilter(x)] diff --git a/packaging/components.nix b/packaging/components.nix index 112791c27d38..fae41caf5084 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -512,8 +512,6 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; - # The clang-tidy plugin is a build-time tool loaded into clang-tidy itself, # so it must be built with a clang stdenv for ABI compatibility with the # clang-tidy binary from the same llvmPackages set, regardless of the diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index a29d65cb2a8d..783818e0ebf0 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -127,7 +127,6 @@ nixComponents.callPackage ( rest = builtins.substring 2 (builtins.stringLength flag) flag; in "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; availableComponents = lib.filterAttrs ( @@ -170,12 +169,7 @@ nixComponents.callPackage ( # perhaps other things that are primarily for overriding and not the shell. config = { # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] - ); + getComponents = c: builtins.removeAttrs c (lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]); }; /** @@ -211,7 +205,6 @@ nixComponents.callPackage ( "nix-fetchers-tests" "nix-flake-tests" "nix-functional-tests" - "nix-perl-bindings" ] (_: null)) c ); }; @@ -287,9 +280,6 @@ nixComponents.callPackage ( ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) - ) ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); @@ -345,8 +335,7 @@ nixComponents.callPackage ( lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark ++ dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + ); propagatedBuildInputs = dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) diff --git a/packaging/everything.nix b/packaging/everything.nix index 751d861c9e80..df7d57a85860 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -41,8 +41,6 @@ nix-internal-api-docs, nix-external-api-docs, - nix-perl-bindings, - testers, patchedSrc ? null, @@ -65,16 +63,7 @@ let nix-main-c nix-cmd ; - } - // - lib.optionalAttrs - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - { - # Currently fails in static build - inherit - nix-perl-bindings - ; - }; + }; devdoc = buildEnv { name = "nix-${nix-cli.version}-devdoc"; @@ -144,13 +133,6 @@ stdenv.mkDerivation (finalAttrs: { lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ nix-util-tests.tests.run-without-new-syscalls - ] - ++ - lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings ]; nativeBuildInputs = [ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index e3f8f1c1f1cb..e30b62eb578d 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -33,7 +33,6 @@ let forAllPackages = forAllPackages' { }; forAllPackages' = { - enableBindings ? false, enableDocs ? false, # already have separate attrs for these }: lib.genAttrs ( @@ -66,9 +65,6 @@ let "nix-json-schema-checks" "nix-clang-tidy-plugin" ] - ++ lib.optionals enableBindings [ - "nix-perl-bindings" - ] ++ lib.optionals enableDocs [ "nix-manual" "nix-manual-manpages-only" @@ -85,7 +81,6 @@ rec { let arbitrarySystem = "x86_64-linux"; listedPkgs = forAllPackages' { - enableBindings = true; enableDocs = true; } (_: null); actualPkgs = lib.concatMapAttrs ( @@ -176,8 +171,6 @@ rec { # Build without unity to catch include issues. withUnityBuild = false; nix-expr = super.nix-expr.override { enableGC = false; }; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = null; } ) ); @@ -201,8 +194,6 @@ rec { pkgs.nixComponents2.overrideScope ( self: super: { withTSan = true; - # Dies at startup. - nix-perl-bindings = null; # TSan has issues with fork and threads. nix-functional-tests = super.nix-functional-tests.overrideAttrs { doCheck = false; }; } @@ -256,9 +247,6 @@ rec { ) (forAllSystems (system: components.${system}.${pkgName})) ); - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); - # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. diff --git a/src/perl/.version b/src/perl/.version deleted file mode 120000 index b7badcd0cc85..000000000000 --- a/src/perl/.version +++ /dev/null @@ -1 +0,0 @@ -../../.version \ No newline at end of file diff --git a/src/perl/.yath.rc.in b/src/perl/.yath.rc.in deleted file mode 100644 index e6f5f93ecdd1..000000000000 --- a/src/perl/.yath.rc.in +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(@lib_dir@) diff --git a/src/perl/MANIFEST b/src/perl/MANIFEST deleted file mode 100644 index 08897647c978..000000000000 --- a/src/perl/MANIFEST +++ /dev/null @@ -1,7 +0,0 @@ -Changes -Makefile.PL -MANIFEST -Nix.xs -README -t/Nix.t -lib/Nix.pm diff --git a/src/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in deleted file mode 100644 index ad51cff3b28b..000000000000 --- a/src/perl/lib/Nix/Config.pm.in +++ /dev/null @@ -1,24 +0,0 @@ -package Nix::Config; - -use MIME::Base64; -use Nix::Store; - -$version = "@PACKAGE_VERSION@"; - -$storeDir = Nix::Store::getStoreDir; - -%config = (); - -sub readConfig { - my $config = "$confDir/nix.conf"; - return unless -f $config; - - open CONFIG, "<$config" or die "cannot open '$config'"; - while () { - /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; - $config{$1} = $2; - } - close CONFIG; -} - -return 1; diff --git a/src/perl/lib/Nix/CopyClosure.pm b/src/perl/lib/Nix/CopyClosure.pm deleted file mode 100644 index 902ee1a1bc9f..000000000000 --- a/src/perl/lib/Nix/CopyClosure.pm +++ /dev/null @@ -1,61 +0,0 @@ -package Nix::CopyClosure; - -use utf8; -use strict; -use Nix::Config; -use Nix::Store; -use Nix::SSH; -use List::Util qw(sum); -use IPC::Open2; - - -sub copyToOpen { - my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; - - $useSubstitutes = 0 if $dryRun || !defined $useSubstitutes; - - # Get the closure of this path. - my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, - map { followLinksToStorePath $_ } @{$storePaths}))); - - # Send the "query valid paths" command with the "lock" option - # enabled. This prevents a race where the remote host - # garbage-collect paths that are already there. Optionally, ask - # the remote host to substitute missing paths. - syswrite($to, pack("L{url} eq $info->{url}; - } - - push @{$narFileList}, $info if !$found; -} - - -sub addPatch { - my ($patches, $storePath, $patch) = @_; - - $$patches{$storePath} = [] - unless defined $$patches{$storePath}; - - my $patchList = $$patches{$storePath}; - - my $found = 0; - foreach my $patch2 (@{$patchList}) { - $found = 1 if - $patch2->{url} eq $patch->{url} && - $patch2->{basePath} eq $patch->{basePath}; - } - - push @{$patchList}, $patch if !$found; - - return !$found; -} - - -sub readManifest_ { - my ($manifest, $addNAR, $addPatch) = @_; - - # Decompress the manifest if necessary. - if ($manifest =~ /\.bz2$/) { - open MANIFEST, "$Nix::Config::bzip2 -d < $manifest |" - or die "cannot decompress '$manifest': $!"; - } else { - open MANIFEST, "<$manifest" - or die "cannot open '$manifest': $!"; - } - - my $inside = 0; - my $type; - - my $manifestVersion = 2; - - my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); - my ($narHash, $narSize, $references, $deriver, $copyFrom, $system, $compressionType); - - while () { - chomp; - s/\#.*$//g; - next if (/^$/); - - if (!$inside) { - - if (/^\s*(\w*)\s*\{$/) { - $type = $1; - $type = "narfile" if $type eq ""; - $inside = 1; - undef $storePath; - undef $url; - undef $hash; - undef $size; - undef $narHash; - undef $narSize; - undef $basePath; - undef $baseHash; - undef $patchType; - undef $system; - $references = ""; - $deriver = ""; - $compressionType = "bzip2"; - } - - } else { - - if (/^\}$/) { - $inside = 0; - - if ($type eq "narfile") { - &$addNAR($storePath, - { url => $url, hash => $hash, size => $size - , narHash => $narHash, narSize => $narSize - , references => $references - , deriver => $deriver - , system => $system - , compressionType => $compressionType - }); - } - - elsif ($type eq "patch") { - &$addPatch($storePath, - { url => $url, hash => $hash, size => $size - , basePath => $basePath, baseHash => $baseHash - , narHash => $narHash, narSize => $narSize - , patchType => $patchType - }); - } - - } - - elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } - elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } - elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } - elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*Compression:\s*(\S+)\s*$/) { $compressionType = $1; } - elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } - elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } - elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } - elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } - elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } - elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; } - elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } - elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } - elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } - elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; } - - # Compatibility; - elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; } - - } - } - - close MANIFEST; - - return $manifestVersion; -} - - -sub readManifest { - my ($manifest, $narFiles, $patches) = @_; - readManifest_($manifest, - sub { addNAR($narFiles, @_); }, - sub { addPatch($patches, @_); } ); -} - - -sub writeManifest { - my ($manifest, $narFiles, $patches, $noCompress) = @_; - - open MANIFEST, ">$manifest.tmp"; # !!! check exclusive - - print MANIFEST "version {\n"; - print MANIFEST " ManifestVersion: 3\n"; - print MANIFEST "}\n"; - - foreach my $storePath (sort (keys %{$narFiles})) { - my $narFileList = $$narFiles{$storePath}; - foreach my $narFile (@{$narFileList}) { - print MANIFEST "{\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $narFile->{url}\n"; - print MANIFEST " Compression: $narFile->{compressionType}\n"; - print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; - print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; - print MANIFEST " NarHash: $narFile->{narHash}\n"; - print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize}; - print MANIFEST " References: $narFile->{references}\n" - if defined $narFile->{references} && $narFile->{references} ne ""; - print MANIFEST " Deriver: $narFile->{deriver}\n" - if defined $narFile->{deriver} && $narFile->{deriver} ne ""; - print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system}; - print MANIFEST "}\n"; - } - } - - foreach my $storePath (sort (keys %{$patches})) { - my $patchList = $$patches{$storePath}; - foreach my $patch (@{$patchList}) { - print MANIFEST "patch {\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $patch->{url}\n"; - print MANIFEST " Hash: $patch->{hash}\n"; - print MANIFEST " Size: $patch->{size}\n"; - print MANIFEST " NarHash: $patch->{narHash}\n"; - print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize}; - print MANIFEST " BasePath: $patch->{basePath}\n"; - print MANIFEST " BaseHash: $patch->{baseHash}\n"; - print MANIFEST " Type: $patch->{patchType}\n"; - print MANIFEST "}\n"; - } - } - - - close MANIFEST; - - rename("$manifest.tmp", $manifest) - or die "cannot rename $manifest.tmp: $!"; - - - # Create a bzipped manifest. - unless (defined $noCompress) { - system("$Nix::Config::bzip2 < $manifest > $manifest.bz2.tmp") == 0 - or die "cannot compress manifest"; - - rename("$manifest.bz2.tmp", "$manifest.bz2") - or die "cannot rename $manifest.bz2.tmp: $!"; - } -} - - -# Return a fingerprint of a store path to be used in binary cache -# signatures. It contains the store path, the base-32 SHA-256 hash of -# the contents of the path, and the references. -sub fingerprintPath { - my ($storePath, $narHash, $narSize, $references) = @_; - die if substr($storePath, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; - die if substr($narHash, 0, 7) ne "sha256:"; - # Convert hash from base-16 to base-32, if necessary. - $narHash = "sha256:" . convertHash("sha256", substr($narHash, 7), 1) - if length($narHash) == 71; - die if length($narHash) != 59; - foreach my $ref (@{$references}) { - die if substr($ref, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; - } - return "1;" . $storePath . ";" . $narHash . ";" . $narSize . ";" . join(",", @{$references}); -} - - -# Parse a NAR info file. -sub parseNARInfo { - my ($storePath, $content, $requireValidSig, $location) = @_; - - my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system, $sig); - my $compression = "bzip2"; - my @refs; - - foreach my $line (split "\n", $content) { - return undef unless $line =~ /^(.*): (.*)$/; - if ($1 eq "StorePath") { $storePath2 = $2; } - elsif ($1 eq "URL") { $url = $2; } - elsif ($1 eq "Compression") { $compression = $2; } - elsif ($1 eq "FileHash") { $fileHash = $2; } - elsif ($1 eq "FileSize") { $fileSize = int($2); } - elsif ($1 eq "NarHash") { $narHash = $2; } - elsif ($1 eq "NarSize") { $narSize = int($2); } - elsif ($1 eq "References") { @refs = split / /, $2; } - elsif ($1 eq "Deriver") { $deriver = $2; } - elsif ($1 eq "System") { $system = $2; } - elsif ($1 eq "Sig") { $sig = $2; } - } - - return undef if $storePath ne $storePath2 || !defined $url || !defined $narHash; - - my $res = - { url => $url - , compression => $compression - , fileHash => $fileHash - , fileSize => $fileSize - , narHash => $narHash - , narSize => $narSize - , refs => [ @refs ] - , deriver => $deriver - , system => $system - }; - - if ($requireValidSig) { - # FIXME: might be useful to support multiple signatures per .narinfo. - - if (!defined $sig) { - warn "NAR info file '$location' lacks a signature; ignoring\n"; - return undef; - } - my ($keyName, $sig64) = split ":", $sig; - return undef unless defined $keyName && defined $sig64; - - my $publicKey = $Nix::Config::binaryCachePublicKeys{$keyName}; - if (!defined $publicKey) { - warn "NAR info file '$location' is signed by unknown key '$keyName'; ignoring\n"; - return undef; - } - - my $fingerprint; - eval { - $fingerprint = fingerprintPath( - $storePath, $narHash, $narSize, - [ map { "$Nix::Config::storeDir/$_" } @refs ]); - }; - if ($@) { - warn "cannot compute fingerprint of '$location'; ignoring\n"; - return undef; - } - - if (!checkSignature($publicKey, decode_base64($sig64), $fingerprint)) { - warn "NAR info file '$location' has an incorrect signature; ignoring\n"; - return undef; - } - - $res->{signedBy} = $keyName; - } - - return $res; -} - - -return 1; diff --git a/src/perl/lib/Nix/SSH.pm b/src/perl/lib/Nix/SSH.pm deleted file mode 100644 index 490ba0ea991e..000000000000 --- a/src/perl/lib/Nix/SSH.pm +++ /dev/null @@ -1,110 +0,0 @@ -package Nix::SSH; - -use utf8; -use strict; -use File::Temp qw(tempdir); -use IPC::Open2; - -our @ISA = qw(Exporter); -our @EXPORT = qw( - @globalSshOpts - readN readInt readString readStrings - writeInt writeString writeStrings - connectToRemoteNix -); - - -our @globalSshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); - - -sub readN { - my ($bytes, $from) = @_; - my $res = ""; - while ($bytes > 0) { - my $s; - my $n = sysread($from, $s, $bytes); - die "I/O error reading from remote side\n" if !defined $n; - die "got EOF while expecting $bytes bytes from remote side\n" if !$n; - $bytes -= $n; - $res .= $s; - } - return $res; -} - - -sub readInt { - my ($from) = @_; - return unpack("L= 0x300; - - return ($from, $to, $pid); -} - - -1; diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm deleted file mode 100644 index f2ae7e88f81d..000000000000 --- a/src/perl/lib/Nix/Store.pm +++ /dev/null @@ -1,45 +0,0 @@ -package Nix::Store; - -use strict; -use warnings; - -require Exporter; - -our @ISA = qw(Exporter); - -our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); - -our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); - -our @EXPORT = qw( - StoreWrapper - StoreWrapper::new - StoreWrapper::isValidPath StoreWrapper::queryReferences StoreWrapper::queryPathInfo StoreWrapper::queryDeriver StoreWrapper::queryPathHash - StoreWrapper::queryPathFromHashPart - StoreWrapper::topoSortPaths StoreWrapper::computeFSClosure followLinksToStorePath StoreWrapper::exportPaths StoreWrapper::importPaths - StoreWrapper::addToStore StoreWrapper::makeFixedOutputPath - StoreWrapper::derivationFromPath - StoreWrapper::addTempRoot - StoreWrapper::queryRawRealisation - - hashPath hashFile hashString convertHash - signString checkSignature - getStoreDir - setVerbosity -); - -our $VERSION = '0.15'; - -sub backtick { - open(RES, "-|", @_) or die; - local $/; - my $res = || ""; - close RES or die; - return $res; -} - -require XSLoader; -XSLoader::load('Nix::Store', $VERSION); - -1; -__END__ diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs deleted file mode 100644 index 8b28b0e5397c..000000000000 --- a/src/perl/lib/Nix/Store.xs +++ /dev/null @@ -1,430 +0,0 @@ -#include "EXTERN.h" -#include "perl.h" -#include "XSUB.h" - -/* Prevent a clash between some Perl and libstdc++ macros. */ -#undef do_open -#undef do_close - -#include "nix/store/derivations.hh" -#include "nix/store/realisation.hh" -#include "nix/store/globals.hh" -#include "nix/store/store-open.hh" -#include "nix/util/posix-source-accessor.hh" -#include "nix/store/export-import.hh" - -#include -#include - -using namespace nix; - -static bool libStoreInitialized = false; - -struct StoreWrapper { - ref store; -}; - -MODULE = Nix::Store PACKAGE = Nix::Store -PROTOTYPES: ENABLE - -TYPEMAP: < _store; - try { - if (!libStoreInitialized) { - initLibStore(); - libStoreInitialized = true; - } - if (items == 1) { - _store = openStore(); - RETVAL = new StoreWrapper { - .store = ref{_store} - }; - } else { - RETVAL = new StoreWrapper { - .store = openStore(s) - }; - } - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void init() - CODE: - if (!libStoreInitialized) { - initLibStore(); - libStoreInitialized = true; - } - - -void setVerbosity(int level) - CODE: - verbosity = (Verbosity) level; - - -int -StoreWrapper::isValidPath(char * path) - CODE: - try { - RETVAL = THIS->store->isValidPath(THIS->store->parseStorePath(path)); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -SV * -StoreWrapper::queryReferences(char * path) - PPCODE: - try { - for (auto & i : THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->references) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathHash(char * path) - PPCODE: - try { - auto s = THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryDeriver(char * path) - PPCODE: - try { - auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); - if (!info->deriver) XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathInfo(char * path, int base32) - PPCODE: - try { - auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); - if (!info->deriver) - XPUSHs(&PL_sv_undef); - else - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - mXPUSHi(info->registrationTime); - mXPUSHi(info->narSize); - AV * refs = newAV(); - for (auto & i : info->references) - av_push(refs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) refs))); - AV * sigs = newAV(); - for (auto & i : info->sigs) - av_push(sigs, newSVpv(i.to_string().c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) sigs))); - } catch (Error & e) { - croak("%s", e.what()); - } - -SV * -StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) - PPCODE: - try { - auto realisation = THIS->store->queryRealisation(DrvOutput{ - .drvPath = THIS->store->parseStorePath(drvPath), - .outputName = outputName, - }); - if (realisation) - XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); - else - XPUSHs(sv_2mortal(newSVpv("", 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathFromHashPart(char * hashPart) - PPCODE: - try { - auto path = THIS->store->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path ? THIS->store->printStorePath(*path).c_str() : "", 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) - PPCODE: - try { - StorePathSet paths; - for (int n = 3; n < items; ++n) - THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); - for (auto & i : paths) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::topoSortPaths(...) - PPCODE: - try { - StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); - auto sorted = THIS->store->topoSortPaths(paths); - for (auto & i : sorted) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::followLinksToStorePath(char * path) - CODE: - try { - RETVAL = newSVpv(THIS->store->printStorePath(THIS->store->followLinksToStorePath(path)).c_str(), 0); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void -StoreWrapper::exportPaths(int fd, ...) - PPCODE: - try { - StorePathSet paths; - for (int n = 2; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); - FdSink sink(fd); - exportPaths(*THIS->store, paths, sink); - } catch (Error & e) { - croak("%s", e.what()); - } - - -void -StoreWrapper::importPaths(int fd, int dontCheckSigs) - PPCODE: - try { - FdSource source(fd); - importPaths(*THIS->store, source, dontCheckSigs ? NoCheckSigs : CheckSigs); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -hashPath(char * algo, int base32, char * path) - PPCODE: - try { - Hash h = hashPath( - makeFSSourceAccessor(absPath(path)), - FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * hashFile(char * algo, int base32, char * path) - PPCODE: - try { - Hash h = hashFile(parseHashAlgo(algo), path); - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * hashString(char * algo, int base32, char * s) - PPCODE: - try { - Hash h = hashString(parseHashAlgo(algo), s); - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * convertHash(char * algo, char * s, int toBase32) - PPCODE: - try { - auto h = Hash::parseAny(s, parseHashAlgo(algo)); - auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * signString(char * secretKey_, char * msg) - PPCODE: - try { - auto sig = SecretKey(secretKey_).signDetached(msg).to_string(); - XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -int checkSignature(SV * publicKey_, SV * sig_, char * msg) - CODE: - try { - STRLEN publicKeyLen; - unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); - if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) - throw Error("public key is not valid"); - - STRLEN sigLen; - unsigned char * sig = (unsigned char *) SvPV(sig_, sigLen); - if (sigLen != crypto_sign_BYTES) - throw Error("signature is not valid"); - - RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -SV * -StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) - PPCODE: - try { - auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; - auto path = THIS->store->addToStore( - std::string(baseNameOf(srcPath)), - makeFSSourceAccessor(absPath(srcPath)), - method, parseHashAlgo(algo)); - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) - PPCODE: - try { - auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; - auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { - .method = method, - .hash = h, - .references = {}, - }); - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::derivationFromPath(char * drvPath) - PREINIT: - HV *hash; - CODE: - try { - Derivation drv = THIS->store->derivationFromPath(THIS->store->parseStorePath(drvPath)); - hash = newHV(); - - HV * outputs = newHV(); - for (auto & i : drv.outputsAndOptPaths(*THIS->store)) { - hv_store( - outputs, i.first.c_str(), i.first.size(), - !i.second.second - ? newSV(0) /* null value */ - : newSVpv(THIS->store->printStorePath(*i.second.second).c_str(), 0), - 0); - } - hv_stores(hash, "outputs", newRV((SV *) outputs)); - - AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs.map) - av_push(inputDrvs, newSVpv(THIS->store->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second - hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); - - AV * inputSrcs = newAV(); - for (auto & i : drv.inputSrcs) - av_push(inputSrcs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); - hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); - - hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); - hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0)); - - AV * args = newAV(); - for (auto & i : drv.args) - av_push(args, newSVpv(i.c_str(), 0)); - hv_stores(hash, "args", newRV((SV *) args)); - - HV * env = newHV(); - for (auto & i : drv.env) - hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0); - hv_stores(hash, "env", newRV((SV *) env)); - - RETVAL = newRV_noinc((SV *)hash); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void -StoreWrapper::addTempRoot(char * storePath) - PPCODE: - try { - THIS->store->addTempRoot(THIS->store->parseStorePath(storePath)); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * getStoreDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir.c_str(), 0))); diff --git a/src/perl/lib/Nix/Utils.pm b/src/perl/lib/Nix/Utils.pm deleted file mode 100644 index 44955a70698c..000000000000 --- a/src/perl/lib/Nix/Utils.pm +++ /dev/null @@ -1,47 +0,0 @@ -package Nix::Utils; - -use utf8; -use File::Temp qw(tempdir); - -our @ISA = qw(Exporter); -our @EXPORT = qw(checkURL uniq writeFile readFile mkTempDir); - -$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )"; - -sub checkURL { - my ($url) = @_; - die "invalid URL '$url'\n" unless $url =~ /^ $urlRE $ /x; -} - -sub uniq { - my %seen; - my @res; - foreach my $name (@_) { - next if $seen{$name}; - $seen{$name} = 1; - push @res, $name; - } - return @res; -} - -sub writeFile { - my ($fn, $s) = @_; - open TMP, ">$fn" or die "cannot create file '$fn': $!"; - print TMP "$s" or die; - close TMP or die; -} - -sub readFile { - local $/ = undef; - my ($fn) = @_; - open TMP, "<$fn" or die "cannot open file '$fn': $!"; - my $s = ; - close TMP or die; - return $s; -} - -sub mkTempDir { - my ($name) = @_; - return tempdir("$name.XXXXXX", CLEANUP => 1, DIR => $ENV{"TMPDIR"} // $ENV{"XDG_RUNTIME_DIR"} // "/tmp") - || die "cannot create a temporary directory"; -} diff --git a/src/perl/lib/Nix/meson.build b/src/perl/lib/Nix/meson.build deleted file mode 100644 index dd5560e21cc5..000000000000 --- a/src/perl/lib/Nix/meson.build +++ /dev/null @@ -1,61 +0,0 @@ -# Nix-Perl Scripts -#============================================================================ - - - -# Sources -#------------------------------------------------- - -nix_perl_store_xs = files('Store.xs') - -nix_perl_scripts = files( - 'CopyClosure.pm', - 'Manifest.pm', - 'SSH.pm', - 'Store.pm', - 'Utils.pm', -) - -nix_perl_scripts_copy_tgts = [] -foreach f : nix_perl_scripts - nix_perl_scripts_copy_tgts += fs.copyfile(f) -endforeach - - -# Targets -#--------------------------------------------------- - -nix_perl_scripts += configure_file( - output : 'Config.pm', - input : 'Config.pm.in', - configuration : nix_perl_conf, -) - -nix_perl_store_cc = custom_target( - 'Store.cc', - output : 'Store.cc', - input : nix_perl_store_xs, - command : [ xsubpp, '@INPUT@', '-output', '@OUTPUT@' ], -) - -# Build Nix::Store Library -#------------------------------------------------- -nix_perl_store_lib = library( - 'Store', - sources : nix_perl_store_cc, - name_prefix : '', - prelink : true, # For C++ static initializers - install : true, - install_mode : 'rwxr-xr-x', - install_dir : join_paths(nix_perl_install_dir, 'auto', 'Nix', 'Store'), - dependencies : nix_perl_store_dep_list, -) - - -# Install Scripts -#--------------------------------------------------- -install_data( - nix_perl_scripts, - install_mode : 'rw-r--r--', - install_dir : join_paths(nix_perl_install_dir, 'Nix'), -) diff --git a/src/perl/meson.build b/src/perl/meson.build deleted file mode 100644 index 59f2a66b8386..000000000000 --- a/src/perl/meson.build +++ /dev/null @@ -1,196 +0,0 @@ -# Nix-Perl Meson build -#============================================================================ - - -# init project -#============================================================================ -project( - 'nix-perl', - 'cpp', - version : files('.version'), - meson_version : '>= 1.1', - license : 'LGPL-2.1-or-later', -) - -# setup env -#------------------------------------------------- -fs = import('fs') -cpp = meson.get_compiler('cpp') -nix_perl_conf = configuration_data() -nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) - - -# set error arguments -#------------------------------------------------- -error_args = [ - '-Wdeprecated-copy', - '-Wdeprecated-declarations', - '-Werror=suggest-override', - '-Werror=unused-result', - '-Wignored-qualifiers', - '-Wno-duplicate-decl-specifier', - '-Wno-literal-suffix', - '-Wno-missing-field-initializers', - '-Wno-non-virtual-dtor', - '-Wno-pedantic', - '-Wno-pointer-bool-conversion', - '-Wno-reserved-user-defined-literal', - '-Wno-unknown-warning-option', - '-Wno-unused-parameter', - '-Wno-unused-variable', - '-Wno-variadic-macros', -] - -add_project_arguments( - cpp.get_supported_arguments(error_args), - language : 'cpp', -) - - -# set install directories -#------------------------------------------------- -prefix = get_option('prefix') -libdir = join_paths(prefix, get_option('libdir')) - -# Dependencies -#============================================================================ - -# Required Programs -#------------------------------------------------- -find_program('xz') -xsubpp = find_program('xsubpp') -perl = find_program('perl') -find_program('curl') -yath = find_program('yath', required : false) - -# Required Libraries -#------------------------------------------------- -bzip2_dep = dependency('bzip2', required : false) -if not bzip2_dep.found() - bzip2_dep = cpp.find_library('bz2') - if not bzip2_dep.found() - error('No "bzip2" pkg-config or "bz2" library found') - endif -endif -curl_dep = dependency('libcurl') -libsodium_dep = dependency('libsodium') - -nix_store_dep = dependency('nix-store') - - -# Finding Perl Headers is a pain. as they do not have -# pkgconfig available, are not in a standard location, -# and are installed into a version folder. Use the -# Perl binary to give hints about perl include dir. -# -# Note that until we have a better solution for this, cross-compiling -# the perl bindings does not appear to be possible. -#------------------------------------------------- -perl_archname = run_command( - perl, - '-e', - 'use Config; print $Config{archname};', - check : true, -).stdout() -perl_version = run_command( - perl, - '-e', - 'use Config; print $Config{version};', - check : true, -).stdout() -perl_archlibexp = run_command( - perl, - '-e', - 'use Config; print $Config{archlibexp};', - check : true, -).stdout() -perl_site_libdir = run_command( - perl, - '-e', - 'use Config; print $Config{installsitearch};', - check : true, -).stdout() -nix_perl_install_dir = join_paths( - libdir, - 'perl5', - 'site_perl', - perl_version, - perl_archname, -) - - -# print perl hints for logs -#------------------------------------------------- -message('Perl archname: @0@'.format(perl_archname)) -message('Perl version: @0@'.format(perl_version)) -message('Perl archlibexp: @0@'.format(perl_archlibexp)) -message('Perl install site: @0@'.format(perl_site_libdir)) -message('Assumed Nix-Perl install dir: @0@'.format(nix_perl_install_dir)) - -# Now find perl modules -#------------------------------------------------- -perl_check_dbi = run_command( - perl, - '-e', - 'use DBI; use DBD::SQLite;', - '-I@0@'.format(get_option('dbi_path')), - '-I@0@'.format(get_option('dbd_sqlite_path')), - check : true, -) - -if perl_check_dbi.returncode() == 2 - error('The Perl modules DBI and/or DBD::SQLite are missing.') -else - message('Found Perl Modules: DBI, DBD::SQLite.') -endif - - - -# declare perl dependency -#------------------------------------------------- -perl_dep = declare_dependency( - dependencies : cpp.find_library( - 'perl', - has_headers : [ - join_paths(perl_archlibexp, 'CORE', 'perl.h'), - join_paths(perl_archlibexp, 'CORE', 'EXTERN.h'), - ], - dirs : [ - join_paths(perl_archlibexp, 'CORE'), - ], - ), - include_directories : join_paths(perl_archlibexp, 'CORE'), -) - -# declare dependencies -#------------------------------------------------- -nix_perl_store_dep_list = [ - perl_dep, - bzip2_dep, - curl_dep, - libsodium_dep, - nix_store_dep, -] - -# # build -# #------------------------------------------------- -lib_dir = join_paths('lib', 'Nix') -subdir(lib_dir) - -if get_option('tests').enabled() - yath_rc_conf = configuration_data() - yath_rc_conf.set('lib_dir', lib_dir) - configure_file( - output : '.yath.rc', - input : '.yath.rc.in', - configuration : yath_rc_conf, - ) - subdir('t') - test( - 'nix-perl-test', - yath, - args : [ 'test' ], - workdir : meson.current_build_dir(), - depends : [ nix_perl_store_lib ] + nix_perl_tests_copy_tgts + nix_perl_scripts_copy_tgts, - ) -endif diff --git a/src/perl/meson.options b/src/perl/meson.options deleted file mode 100644 index 03ddf57f1481..000000000000 --- a/src/perl/meson.options +++ /dev/null @@ -1,30 +0,0 @@ -# Nix-Perl build options -#============================================================================ - - -# compiler args -#============================================================================ - -option( - 'tests', - type : 'feature', - value : 'disabled', - description : 'run nix-perl tests', -) - - -# Location of Perl Modules -#============================================================================ -option( - 'dbi_path', - type : 'string', - value : '/usr', - description : 'path to perl::dbi', -) - -option( - 'dbd_sqlite_path', - type : 'string', - value : '/usr', - description : 'path to perl::dbd-SQLite', -) diff --git a/src/perl/package.nix b/src/perl/package.nix deleted file mode 100644 index e25b2996c83c..000000000000 --- a/src/perl/package.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ - lib, - stdenv, - mkMesonDerivation, - pkg-config, - perl, - perlPackages, - nix-store, - version, - curl, - bzip2, - libsodium, -}: - -let - inherit (lib) fileset; -in - -perl.pkgs.toPerlModule ( - mkMesonDerivation (finalAttrs: { - pname = "nix-perl"; - inherit version; - - workDir = ./.; - fileset = fileset.unions ( - [ - ./.version - ../../.version - ./MANIFEST - ./lib - ./meson.build - ./meson.options - ] - ++ lib.optionals finalAttrs.finalPackage.doCheck [ - ./.yath.rc.in - ./t - ] - ); - - nativeBuildInputs = [ - pkg-config - perl - curl - ]; - - buildInputs = [ - nix-store - bzip2 - libsodium - perlPackages.DBI - perlPackages.DBDSQLite - ]; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - preConfigure = - # "Inline" .version so its not a symlink, and includes the suffix - '' - chmod u+w .version - echo ${finalAttrs.version} > .version - ''; - - mesonFlags = [ - (lib.mesonEnable "tests" finalAttrs.finalPackage.doCheck) - ]; - - mesonCheckFlags = [ - "--print-errorlogs" - ]; - - strictDeps = false; - - meta = { - platforms = lib.platforms.unix; - }; - }) -) diff --git a/src/perl/t/init.t b/src/perl/t/init.t deleted file mode 100644 index 80197e013766..000000000000 --- a/src/perl/t/init.t +++ /dev/null @@ -1,13 +0,0 @@ -use strict; -use warnings; -use Test2::V0; - -use Nix::Store; - -my $s = new Nix::Store("dummy://"); - -my $res = $s->isValidPath("/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); - -ok(!$res, "should not have path"); - -done_testing; diff --git a/src/perl/t/meson.build b/src/perl/t/meson.build deleted file mode 100644 index f95bee2ffc37..000000000000 --- a/src/perl/t/meson.build +++ /dev/null @@ -1,15 +0,0 @@ -# Nix-Perl Tests -#============================================================================ - - -# src -#--------------------------------------------------- - -nix_perl_tests = files( - 'init.t', -) - -nix_perl_tests_copy_tgts = [] -foreach f : nix_perl_tests - nix_perl_tests_copy_tgts += fs.copyfile(f) -endforeach diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index 7abadd43af64..b61a3929af80 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -1,26 +1,9 @@ -{ nixComponents, ... }: { name = "fetchers-substitute"; nodes.substituter = { pkgs, ... }: { - # nix-serve is broken while cross-compiling in nixpkgs 25.11. It's been - # fixed since, but while we're pinning 25.11 we use this workaround. - nixpkgs.overlays = [ - (final: prev: { - nix-serve = - final.lib.warnIf (final.lib.versions.majorMinor final.lib.version != "25.11") - "remove the hack in fetchers-substitute.nix when updating nixpkgs from 25.11" - ( - prev.nix-serve.override { - nix = prev.nix // { - libs.nix-perl-bindings = nixComponents.nix-perl-bindings; - }; - } - ); - }) - ]; virtualisation.writableStore = true; nix.settings.extra-experimental-features = [ @@ -30,6 +13,7 @@ networking.firewall.allowedTCPPorts = [ 5000 ]; + # TODO stop using this, because it has to depend on an older version of Nix that still has the perl bindings. services.nix-serve = { enable = true; secretKeyFile = From 5172f041353f0b26454264fbc5e163518f76a956 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Fri, 1 May 2026 20:35:35 +0200 Subject: [PATCH 089/364] Deduplicate terminal realisation queries in CA derivation resolution This change introduces RealisationCache to ensure each unique derivation output is only queried once from the store during graph traversal. Internal helper functions are moved into an anonymous namespace. --- src/libstore/include/nix/store/realisation.hh | 21 +++ src/libstore/outputs-query.cc | 131 ++++++++++++++---- 2 files changed, 125 insertions(+), 27 deletions(-) diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 33159b6dc848..ef89d290aa62 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -4,6 +4,7 @@ #include #include "nix/util/hash.hh" +#include "nix/util/std-hash.hh" #include "nix/store/path.hh" #include "nix/store/derived-path.hh" #include @@ -178,6 +179,26 @@ public: } // namespace nix +template<> +struct std::hash +{ + std::size_t operator()(const nix::DrvOutput & id) const noexcept + { + std::size_t h = 0; + nix::hash_combine(h, id.drvPath, id.outputName); + return h; + } +}; + +namespace nix { + +inline std::size_t hash_value(const DrvOutput & id) +{ + return std::hash{}(id); +} + +} // namespace nix + JSON_IMPL(nix::DrvOutput) JSON_IMPL(nix::UnkeyedRealisation) JSON_IMPL(nix::Realisation) diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index 531bcc85521e..b7bd97d0ed99 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -3,8 +3,26 @@ #include "nix/store/realisation.hh" #include "nix/util/util.hh" +#include + namespace nix { +namespace { + +/** + * Cache mapping a resolved derivation output to its realisation output path. + */ +using RealisationCache = boost::unordered_flat_map>; + +/* Forward declaration so resolveSingleDerivedPath can call it. */ +static std::optional deepQueryPartialDerivationOutputImpl( + Store & store, + const StorePath & drvPath, + const std::string & outputName, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache); + /** * Resolve a `SingleDerivedPath` to a concrete store path. * @@ -15,16 +33,21 @@ namespace nix { * @param queryRealisation must already be initialized (not empty) */ static std::optional resolveSingleDerivedPath( - Store & store, const SingleDerivedPath & path, Store * evalStore_, QueryRealisationFun & queryRealisation) + Store & store, + const SingleDerivedPath & path, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) { return std::visit( overloaded{ [](const SingleDerivedPath::Opaque & opaque) -> std::optional { return opaque.path; }, [&](const SingleDerivedPath::Built & built) -> std::optional { - auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation); + auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, resCache); if (!innerPath) return std::nullopt; - return deepQueryPartialDerivationOutput(store, *innerPath, built.output, evalStore_, queryRealisation); + return deepQueryPartialDerivationOutputImpl( + store, *innerPath, built.output, evalStore_, queryRealisation, resCache); }, }, path.raw()); @@ -35,8 +58,12 @@ static std::optional resolveSingleDerivedPath( * * @param queryRealisation must already be initialized (not empty) */ -static std::pair -resolveDerivation(Store & store, const StorePath & drvPath, Store * evalStore_, QueryRealisationFun & queryRealisation) +static std::pair resolveDerivation( + Store & store, + const StorePath & drvPath, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) { auto & evalStore = evalStore_ ? *evalStore_ : store; @@ -50,11 +77,12 @@ resolveDerivation(Store & store, const StorePath & drvPath, Store * evalStore_, store, [&](ref depDrvPath, const std::string & depOutputName) -> std::optional { - auto concreteDrvPath = resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation); + auto concreteDrvPath = + resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, resCache); if (!concreteDrvPath) return std::nullopt; - return deepQueryPartialDerivationOutput( - store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation); + return deepQueryPartialDerivationOutputImpl( + store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, resCache); }); if (resolvedDrv) drv = Derivation{*resolvedDrv}; @@ -69,21 +97,79 @@ void queryPartialDerivationOutputMapCA( const StorePath & drvPath, const BasicDerivation & drv, std::map> & outputs, - QueryRealisationFun queryRealisation) + QueryRealisationFun queryRealisation, + RealisationCache & resCache) { if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; for (auto & [outputName, _] : drv.outputs) { - auto realisation = queryRealisation(DrvOutput{drvPath, outputName}); - if (realisation) { - outputs.insert_or_assign(outputName, realisation->outPath); + DrvOutput id{drvPath, outputName}; + auto it = resCache.find(id); + if (it != resCache.end()) { + outputs.insert_or_assign(outputName, it->second); + continue; + } + + auto realisation = queryRealisation(id); + std::optional outPath = realisation ? std::optional{realisation->outPath} : std::nullopt; + resCache.emplace(id, outPath); + + if (outPath) { + outputs.insert_or_assign(outputName, *outPath); } else { outputs.insert({outputName, std::nullopt}); } } } +/** + * Internal implementation of deepQueryPartialDerivationOutput that accepts a + * shared RealisationCache, allowing memoization of realisation queries across recursive calls. + */ +static std::optional deepQueryPartialDerivationOutputImpl( + Store & store, + const StorePath & drvPath, + const std::string & outputName, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + auto staticResult = evalStore.queryStaticPartialDerivationOutput(drvPath, outputName); + if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return staticResult; + + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + + if (drv.outputs.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); + + DrvOutput id{resolvedDrvPath, outputName}; + auto it = resCache.find(id); + if (it != resCache.end()) + return it->second; + + auto realisation = queryRealisation(id); + std::optional outPath = realisation ? std::optional{realisation->outPath} : std::nullopt; + resCache.emplace(id, outPath); + return outPath; +} + +} // namespace + +void queryPartialDerivationOutputMapCA( + Store & store, + const StorePath & drvPath, + const BasicDerivation & drv, + std::map> & outputs, + QueryRealisationFun queryRealisation) +{ + RealisationCache resCache; + queryPartialDerivationOutputMapCA(store, drvPath, drv, outputs, queryRealisation, resCache); +} + std::map> deepQueryPartialDerivationOutputMap( Store & store, const StorePath & drvPath, Store * evalStore_, QueryRealisationFun queryRealisation) { @@ -97,8 +183,9 @@ std::map> deepQueryPartialDerivationOutput if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation); - queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation); + RealisationCache resCache; + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation, resCache); return outputs; } @@ -123,22 +210,12 @@ std::optional deepQueryPartialDerivationOutput( Store * evalStore_, QueryRealisationFun queryRealisation) { - auto & evalStore = evalStore_ ? *evalStore_ : store; - if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; - auto staticResult = evalStore.queryStaticPartialDerivationOutput(drvPath, outputName); - if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return staticResult; - - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation); - - if (drv.outputs.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); - - auto realisation = queryRealisation(DrvOutput{resolvedDrvPath, outputName}); - return realisation ? std::optional{realisation->outPath} : std::nullopt; + RealisationCache resCache; + return deepQueryPartialDerivationOutputImpl( + store, drvPath, outputName, evalStore_, queryRealisation, resCache); } } // namespace nix From e094bbd1cfba091a8cf973d540122cb1a41ca1a5 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Fri, 1 May 2026 20:35:38 +0200 Subject: [PATCH 090/364] Fix exponential complexity in CA derivation output resolution By memoizing the structural resolution of derivations in ResolveCache, we restore linear O(N) performance for graph traversal. This prevents the O(Fib(N)) blowup previously seen in graphs with many overlapping dependencies. --- src/libstore/outputs-query.cc | 45 +++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index b7bd97d0ed99..6a5cce222339 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -9,6 +9,13 @@ namespace nix { namespace { +/** + * Cache mapping an unresolved drv path to its resolved (Derivation, StorePath) + * pair. Shared across a single top-level resolution call to prevent exponential + * re-traversal of the closure when many derivations share the same dependencies. + */ +using ResolveCache = boost::unordered_flat_map>; + /** * Cache mapping a resolved derivation output to its realisation output path. */ @@ -21,6 +28,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( const std::string & outputName, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache); /** @@ -37,24 +45,31 @@ static std::optional resolveSingleDerivedPath( const SingleDerivedPath & path, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { return std::visit( overloaded{ [](const SingleDerivedPath::Opaque & opaque) -> std::optional { return opaque.path; }, [&](const SingleDerivedPath::Built & built) -> std::optional { - auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, resCache); + auto innerPath = + resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, cache, resCache); if (!innerPath) return std::nullopt; return deepQueryPartialDerivationOutputImpl( - store, *innerPath, built.output, evalStore_, queryRealisation, resCache); + store, *innerPath, built.output, evalStore_, queryRealisation, cache, resCache); }, }, path.raw()); } /** - * Resolve a derivation and compute its store path. + * Resolve a derivation and compute its store path, with memoization. + * + * Results are stored in `cache` (keyed on the unresolved `drvPath`) so that + * each derivation in the closure is resolved at most once per top-level call, + * preventing the exponential re-traversal that would otherwise occur for + * content-addressed derivation closures. * * @param queryRealisation must already be initialized (not empty) */ @@ -63,8 +78,13 @@ static std::pair resolveDerivation( const StorePath & drvPath, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { + auto it = cache.find(drvPath); + if (it != cache.end()) + return it->second; + auto & evalStore = evalStore_ ? *evalStore_ : store; Derivation drv = evalStore.readInvalidDerivation(drvPath); @@ -78,18 +98,20 @@ static std::pair resolveDerivation( [&](ref depDrvPath, const std::string & depOutputName) -> std::optional { auto concreteDrvPath = - resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, resCache); + resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, cache, resCache); if (!concreteDrvPath) return std::nullopt; return deepQueryPartialDerivationOutputImpl( - store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, resCache); + store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, cache, resCache); }); if (resolvedDrv) drv = Derivation{*resolvedDrv}; } auto resolvedDrvPath = computeStorePath(store, drv); - return {std::move(drv), std::move(resolvedDrvPath)}; + auto result = std::make_pair(drv, resolvedDrvPath); + cache.emplace(drvPath, result); + return result; } void queryPartialDerivationOutputMapCA( @@ -125,7 +147,7 @@ void queryPartialDerivationOutputMapCA( /** * Internal implementation of deepQueryPartialDerivationOutput that accepts a - * shared RealisationCache, allowing memoization of realisation queries across recursive calls. + * shared ResolveCache and RealisationCache, allowing memoization across recursive calls. */ static std::optional deepQueryPartialDerivationOutputImpl( Store & store, @@ -133,6 +155,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( const std::string & outputName, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { auto & evalStore = evalStore_ ? *evalStore_ : store; @@ -141,7 +164,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return staticResult; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, cache, resCache); if (drv.outputs.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); @@ -183,8 +206,9 @@ std::map> deepQueryPartialDerivationOutput if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; + ResolveCache cache; RealisationCache resCache; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, cache, resCache); queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation, resCache); return outputs; @@ -213,9 +237,10 @@ std::optional deepQueryPartialDerivationOutput( if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; + ResolveCache cache; RealisationCache resCache; return deepQueryPartialDerivationOutputImpl( - store, drvPath, outputName, evalStore_, queryRealisation, resCache); + store, drvPath, outputName, evalStore_, queryRealisation, cache, resCache); } } // namespace nix From 94e24ddea9d2e8fb9099a95e8f8c0ea4f9d9bff7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 23:50:00 +0300 Subject: [PATCH 091/364] libutil: Fix path traversal in unpackTarfile Fixes GHSA-gr92-w2r5-qw5p. The primary fix is .relative_path() calls, everything else is making sure we use native path handling in libarchive. --- src/libutil/tarfile.cc | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index eea03766375c..b61710c7e0f9 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -5,6 +5,7 @@ #include "nix/util/serialise.hh" #include "nix/util/tarfile.hh" #include "nix/util/file-system.hh" +#include "nix/util/os-string.hh" namespace nix { @@ -123,6 +124,12 @@ TarArchive::~TarArchive() archive_read_free(this->archive); } +#ifndef _WIN32 +# define NIX_LIBARCHIVE_NATIVE_PATH_FUNC(func) func +#else +# define NIX_LIBARCHIVE_NATIVE_PATH_FUNC(func) func##_w +#endif + static void extract_archive(TarArchive & archive, const std::filesystem::path & destDir) { int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -132,24 +139,34 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & int r = archive_read_next_header(archive.archive, &entry); if (r == ARCHIVE_EOF) break; - auto name = archive_entry_pathname(entry); - if (!name) - throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); - if (r == ARCHIVE_WARN) - warn("getting archive member '%1%': %2%", name, archive_error_string(archive.archive)); - else - archive.check(r); - archive_entry_copy_pathname(entry, (destDir / name).string().c_str()); + const auto relPath = [&]() -> std::filesystem::path { + /* Some archives might lack a pathname https://github.com/libarchive/libarchive/issues/2089. */ + auto * name = NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_pathname)(entry); + if (!name) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) + warn( + "getting archive member '%1%': %2%", + os_string_to_string(OsStringView(name)), + archive_error_string(archive.archive)); + else + archive.check(r); + + return std::filesystem::path(name).relative_path(); + }(); + + NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_copy_pathname)(entry, (destDir / relPath).c_str()); // sources can and do contain dirs with no rx bits if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); // Patch hardlink path - const char * original_hardlink = archive_entry_hardlink(entry); - if (original_hardlink) { - archive_entry_copy_hardlink(entry, (destDir / original_hardlink).string().c_str()); + const auto * originalHardlink = NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_hardlink)(entry); + if (originalHardlink) { + auto hardlinkPath = std::filesystem::path(originalHardlink).relative_path(); + NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_copy_hardlink)(entry, (destDir / hardlinkPath).c_str()); } archive.check(archive_read_extract(archive.archive, entry, flags)); @@ -158,6 +175,8 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & archive.close(); } +#undef NIX_LIBARCHIVE_NATIVE_PATH_FUNC + void unpackTarfile(const std::filesystem::path & tarFile, const std::filesystem::path & destDir) { auto archive = TarArchive(tarFile); From e29ca2f114e21ffccaf0de523fe798b6930ee706 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 24 Apr 2025 11:51:10 -0400 Subject: [PATCH 092/364] Simplify the Meson now that upstream bug is fixed Upstream bug is https://github.com/mesonbuild/meson/issues/13584. This has been fixed in https://github.com/mesonbuild/meson/commit/fc9fd42899e1e2160a69ec245931c3aa79b0d267, which is available since Meson 1.8. We now have it with nixpkgs 25.11 and distros should already be caught up. Also does a minor spring clean of our meson accordingly. I noticed that distros have a tendency to patch out our unconditional nix-function-tests subproject from the top-level project, so this also adds an option to disable it. --- doc/manual/meson.build | 2 +- meson.build | 19 +++++++++---------- meson.options | 7 +++++++ src/clang-tidy-plugin/meson.build | 2 +- src/external-api-docs/meson.build | 2 +- src/internal-api-docs/meson.build | 2 +- src/json-schema-checks/meson.build | 2 +- src/libcmd/meson.build | 2 +- src/libexpr-c/meson.build | 2 +- src/libexpr-test-support/meson.build | 2 +- src/libexpr-tests/meson.build | 2 +- src/libexpr/meson.build | 2 +- src/libfetchers-c/meson.build | 2 +- src/libfetchers-tests/meson.build | 2 +- src/libfetchers/meson.build | 2 +- src/libflake-c/meson.build | 2 +- src/libflake-tests/meson.build | 2 +- src/libflake/meson.build | 2 +- src/libmain-c/meson.build | 2 +- src/libmain/meson.build | 2 +- src/libstore-c/meson.build | 2 +- src/libstore-test-support/meson.build | 2 +- src/libstore-tests/meson.build | 2 +- src/libstore/meson.build | 13 +++---------- src/libutil-c/meson.build | 2 +- src/libutil-test-support/meson.build | 2 +- src/libutil-tests/meson.build | 2 +- src/libutil/meson.build | 2 +- src/nix/meson.build | 7 +------ src/nswrapper/meson.build | 3 +-- 30 files changed, 46 insertions(+), 53 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fd841e80cbb..5f1fe2ed6732 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -1,7 +1,7 @@ project( 'nix-manual', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/meson.build b/meson.build index b5d9434a1a84..f2a062e396d8 100644 --- a/meson.build +++ b/meson.build @@ -1,15 +1,12 @@ -# This is just a stub project to include all the others as subprojects -# for development shell purposes +# This is just a top-level project to include all the others as subprojects +# for development shell purposes (when building via Nix) or for distro packaging purposes. project( - 'nix-dev-shell', + 'Nix', 'cpp', version : files('.version'), subproject_dir : 'src', - default_options : [ - 'localstatedir=/nix/var', - ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', ) # Internal Libraries @@ -45,8 +42,6 @@ subproject('libexpr-c') subproject('libflake-c') subproject('libmain-c') -asan_enabled = 'address' in get_option('b_sanitize') - # Testing if get_option('unit-tests') subproject('libutil-test-support') @@ -58,7 +53,11 @@ if get_option('unit-tests') subproject('libexpr-tests') subproject('libflake-tests') endif -subproject('nix-functional-tests') + +if get_option('functional-tests') + subproject('nix-functional-tests') +endif + if get_option('json-schema-checks') subproject('json-schema-checks') endif diff --git a/meson.options b/meson.options index 7b847beba831..2e0d873fae08 100644 --- a/meson.options +++ b/meson.options @@ -14,6 +14,13 @@ option( description : 'Build unit tests', ) +option( + 'functional-tests', + type : 'boolean', + value : true, + description : 'Build functional (E2E) tests', +) + option( 'benchmarks', type : 'boolean', diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build index 60cfd1514912..7896bd085479 100644 --- a/src/clang-tidy-plugin/meson.build +++ b/src/clang-tidy-plugin/meson.build @@ -6,7 +6,7 @@ project( 'cpp_std=c++23', 'warning_level=2', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index 1903b36e589b..d96da3863e6f 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-external-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index 844cb262ee38..3976e4c3f81a 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-internal-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 8a0bde04b8ed..66dc9b758a44 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -6,7 +6,7 @@ project( 'nix-json-schema-checks', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index d970a8e4b066..9638b491f2c4 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index c47704ce4112..c4ca08a34fd3 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index df28661b7e78..4a87bb4545fd 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c20654..92e550a0527b 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 510b6d696e3b..d44a0965d96b 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index db415d9173e7..58f0c26dbffb 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956b9..b9664eaa1611 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index d34dd4f434d1..ed52e565279d 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index fddb39bdf96b..01fc3e0f4f40 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 3512be10bce9..00b592195c2c 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 58916ecd9ab2..c06bf6ba5450 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 36332fdb70a1..8b144c8aa9cf 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 2ac59924e592..0084643bdaad 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index c81235bf16d4..542b9a1a94f9 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 4d904cb1d06a..ca451db4abce 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index ab03bb2d2a0e..a7353f1ec7dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 753c786876ab..0bd42969d8aa 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -9,7 +9,7 @@ project( 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -220,15 +220,14 @@ path_opts = [ 'libdir', 'includedir', 'libexecdir', + 'localstatedir', # Homecooked Nix directories. 'store-dir', - 'localstatedir', 'log-dir', ] # For your grepping pleasure, this loop sets the following variables that aren't mentioned # literally above: # store_dir -# localstatedir # log_dir # profile_dir foreach optname : path_opts @@ -399,13 +398,7 @@ libraries_private = [] extra_pkg_config_variables = { 'storedir' : get_option('store-dir'), + 'localstatedir' : get_option('localstatedir'), } -# Working around https://github.com/mesonbuild/meson/issues/13584 -if host_machine.system() != 'darwin' - extra_pkg_config_variables += { - 'localstatedir' : get_option('localstatedir'), - } -endif - subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 1806dbb6f9a0..c454853c46b2 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 64231107eb6b..7299cd65c328 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 6a86504ded42..82322b9cc31b 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index d132ce67c748..4c801ffa6762 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/nix/meson.build b/src/nix/meson.build index 3327b846c9c4..93200164957d 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -7,7 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'localstatedir=/nix/var', ], meson_version : '>= 1.4', license : 'LGPL-2.1-or-later', @@ -262,11 +261,7 @@ custom_target( # TODO(Ericson3214): Doesn't yet work #meson.override_find_program(linkname, t) -localstatedir = nix_store.get_variable( - 'localstatedir', - default_value : get_option('localstatedir'), -) -assert(localstatedir == get_option('localstatedir')) +localstatedir = nix_store.get_variable('localstatedir') store_dir = nix_store.get_variable('storedir') subdir('scripts') subdir('misc') diff --git a/src/nswrapper/meson.build b/src/nswrapper/meson.build index 77b96d677e72..1d1ccc6b21c3 100644 --- a/src/nswrapper/meson.build +++ b/src/nswrapper/meson.build @@ -7,9 +7,8 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) From 2585efd3aa42814b225680f72b33b8eca677ad3a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 3 May 2026 18:39:44 +0300 Subject: [PATCH 093/364] tests/functional: Migrate more repl tests into characterisation framework This is a much more maintainable and easy way of writing tests. I've dropped some filtering (like the number of variables loaded), since we want to test this too and the whole point of characterisation testing is to avoid the manual churn, so it's not an issue. Also added a way to add comments to the repl test without it affecting the output (special `# COM:` prefix in the input, so that tests can be documented). --- tests/functional/repl.sh | 220 +----------------- .../repl/add-lots-of-variables.expected | 32 +++ .../functional/repl/add-lots-of-variables.in | 2 + .../repl/add-overwritten-symbol-env.expected | 13 ++ .../repl/add-overwritten-symbol-env.in | 4 + .../repl/add-variable-with-spaces.expected | 12 + .../repl/add-variable-with-spaces.in | 3 + tests/functional/repl/attribute-set.nix | 6 + .../repl/doc-comment-curried-args.expected | 3 +- .../repl/doc-comment-formals.expected | 3 +- tests/functional/repl/doc-compact.expected | 3 +- tests/functional/repl/doc-constant.expected | 3 +- tests/functional/repl/doc-floatedIn.expected | 3 +- tests/functional/repl/doc-functor.expected | 3 +- .../repl/doc-lambda-flavors.expected | 3 +- .../functional/repl/doc-measurement.expected | 3 +- tests/functional/repl/doc-multiply.expected | 3 +- .../functional/repl/doc-unambiguous.expected | 3 +- .../functional/repl/dollar-escaping.expected | 5 + tests/functional/repl/dollar-escaping.in | 2 + tests/functional/repl/file-a.nix | 1 + tests/functional/repl/file-b.nix | 1 + .../repl/inherit-and-assignment.expected | 9 + .../functional/repl/inherit-and-assignment.in | 4 + .../repl/inherit-current-scope.expected | 9 + .../functional/repl/inherit-current-scope.in | 4 + .../repl/inherit-missing-shows-pos.expected | 13 ++ .../repl/inherit-missing-shows-pos.in | 4 + .../repl/inherit-multiple-attrs.expected | 9 + .../functional/repl/inherit-multiple-attrs.in | 4 + .../repl/inherit-with-semicolon.expected | 9 + .../functional/repl/inherit-with-semicolon.in | 4 + tests/functional/repl/inherit.expected | 9 + tests/functional/repl/inherit.in | 4 + .../repl/list-loaded-nothing-loaded.expected | 5 + .../repl/list-loaded-nothing-loaded.in | 1 + .../repl/multiple-bindings-same-line.expected | 7 + .../repl/multiple-bindings-same-line.in | 3 + .../functional/repl/nested-attr-path.expected | 7 + tests/functional/repl/nested-attr-path.in | 3 + .../repl/pretty-print-idempotent.expected | 3 +- tests/functional/repl/printing.expected | 59 +++++ tests/functional/repl/printing.in | 11 + .../reload-with-non-existent-file.expected | 24 ++ .../repl/reload-with-non-existent-file.in | 5 + 45 files changed, 315 insertions(+), 226 deletions(-) create mode 100644 tests/functional/repl/add-lots-of-variables.expected create mode 100644 tests/functional/repl/add-lots-of-variables.in create mode 100644 tests/functional/repl/add-overwritten-symbol-env.expected create mode 100644 tests/functional/repl/add-overwritten-symbol-env.in create mode 100644 tests/functional/repl/add-variable-with-spaces.expected create mode 100644 tests/functional/repl/add-variable-with-spaces.in create mode 100644 tests/functional/repl/attribute-set.nix create mode 100644 tests/functional/repl/dollar-escaping.expected create mode 100644 tests/functional/repl/dollar-escaping.in create mode 100644 tests/functional/repl/file-a.nix create mode 100644 tests/functional/repl/file-b.nix create mode 100644 tests/functional/repl/inherit-and-assignment.expected create mode 100644 tests/functional/repl/inherit-and-assignment.in create mode 100644 tests/functional/repl/inherit-current-scope.expected create mode 100644 tests/functional/repl/inherit-current-scope.in create mode 100644 tests/functional/repl/inherit-missing-shows-pos.expected create mode 100644 tests/functional/repl/inherit-missing-shows-pos.in create mode 100644 tests/functional/repl/inherit-multiple-attrs.expected create mode 100644 tests/functional/repl/inherit-multiple-attrs.in create mode 100644 tests/functional/repl/inherit-with-semicolon.expected create mode 100644 tests/functional/repl/inherit-with-semicolon.in create mode 100644 tests/functional/repl/inherit.expected create mode 100644 tests/functional/repl/inherit.in create mode 100644 tests/functional/repl/list-loaded-nothing-loaded.expected create mode 100644 tests/functional/repl/list-loaded-nothing-loaded.in create mode 100644 tests/functional/repl/multiple-bindings-same-line.expected create mode 100644 tests/functional/repl/multiple-bindings-same-line.in create mode 100644 tests/functional/repl/nested-attr-path.expected create mode 100644 tests/functional/repl/nested-attr-path.in create mode 100644 tests/functional/repl/printing.expected create mode 100644 tests/functional/repl/printing.in create mode 100644 tests/functional/repl/reload-with-non-existent-file.expected create mode 100644 tests/functional/repl/reload-with-non-existent-file.in diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 88d6e91cd90f..9e752337f10d 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -120,83 +120,6 @@ testReplResponseNoRegex () { testReplResponseGeneral --fixed-strings "$@" } -# :a uses the newest version of a symbol -# -# shellcheck disable=SC2016 -testReplResponse ' -:a { a = "1"; } -:a { a = "2"; } -"result: ${a}" -' "result: 2" - -# check dollar escaping https://github.com/NixOS/nix/issues/4909 -# note the escaped \, -# \\ -# because the second argument is a regex -# -# shellcheck disable=SC2016 -testReplResponseNoRegex ' -"$" + "{hi}" -' '"\${hi}"' - -# Test inherit statement support (issue #15053) -testReplResponseNoRegex ' -a = { b = 1; c = 2; } -inherit (a) b -b -' '1' - -# inherit multiple attributes -testReplResponseNoRegex ' -a = { x = 10; y = 20; } -inherit (a) x y -x + y -' '30' - -# inherit from current scope -testReplResponseNoRegex ' -foo = 42 -inherit foo -foo -' '42' - -# inherit with semicolon (also works) -testReplResponseNoRegex ' -a = { z = 99; } -inherit (a) z; -z -' '99' - -# multiple bindings on one line -testReplResponseNoRegex ' -a = 1; b = 2; -a + b -' '3' - -# nested attribute path -testReplResponseNoRegex ' -a.b.c = 1; -a.b -' '{ c = 1; }' - -# mixed bindings: inherit and assignment -testReplResponseNoRegex ' -x = { p = 10; } -inherit (x) p; q = 20; -p + q -' '30' - -# inherit error shows position (without spurious semicolon from retry) -testReplResponse ' -a = { x = 1; } -inherit (a) y -y -' "error: attribute 'y' missing -.*at .string.:1:13: -.*inherit (a) y -.* \\^ -.*Did you mean x" - testReplResponse ' drvPath ' '".*-simple.drv"' \ @@ -222,32 +145,6 @@ foo + baz ' "3" \ ./flake ./flake\#bar --experimental-features 'flakes' -testReplResponse $' -:a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } -' 'Added 4 variables. -a, b, longerName, "with spaces" -' - -cat < attribute-set.nix -{ - a = 1; - b = 2; - longerName = 3; - "with spaces" = 4; -} -EOF -testReplResponse ' -:l ./attribute-set.nix -' 'Added 4 variables. -a, b, longerName, "with spaces" -' - -testReplResponseNoRegex $' -:a builtins.foldl\' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) -' 'Added 23 variables. -"0", "1", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "2", "20", "21", "22", "3", "4", "5", "6" -... and 3 more; view with :ll' - # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` # - Modify the flake @@ -313,22 +210,8 @@ EOF grep -q "afterChange" repl_output || fail ":reload didn't pick up git work tree change" fi -# Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, +# Regression: a failed `:lf` must not be remembered for `:reload`, # and an error in one loaded file must not drop later ones from the reload list. -cat > reloadA.nix < reloadB.nix <@g" \ - -e "/Added [0-9]* variables/{s@ [0-9]* @ @;n;d}" \ - -e '/\.\.\. and [0-9]* more; view with :ll/d' \ | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ ; } @@ -500,7 +287,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do read -r -a flags < "$testDir/repl/$test.flags" fi - (cd "$testDir/repl"; set +x; runRepl "${flags[@]}" 2>&1) < "$in" > "$actual" || { + # Allow putting comments (lines starting with `# COM:`) in the test for + # documentation purposes. Regular comments are not skipped, since those are + # also interpreted by the repl. + (cd "$testDir/repl"; set +x; runRepl "${flags[@]}" 2>&1) < <(grep -Ev '^[[:space:]]*#[[:space:]]*COM:' "$in") > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 badExitCode=1 } diff --git a/tests/functional/repl/add-lots-of-variables.expected b/tests/functional/repl/add-lots-of-variables.expected new file mode 100644 index 000000000000..4ba282263b47 --- /dev/null +++ b/tests/functional/repl/add-lots-of-variables.expected @@ -0,0 +1,32 @@ +Nix +Type :? for help. + +nix-repl> :a builtins.foldl' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) +Added 23 variables. +"0", "1", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "2", "20", "21", "22", "3", "4", "5", "6" +... and 3 more; view with :ll + +nix-repl> :ll +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 diff --git a/tests/functional/repl/add-lots-of-variables.in b/tests/functional/repl/add-lots-of-variables.in new file mode 100644 index 000000000000..77796389dfdd --- /dev/null +++ b/tests/functional/repl/add-lots-of-variables.in @@ -0,0 +1,2 @@ +:a builtins.foldl' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) +:ll diff --git a/tests/functional/repl/add-overwritten-symbol-env.expected b/tests/functional/repl/add-overwritten-symbol-env.expected new file mode 100644 index 000000000000..026b7cceb602 --- /dev/null +++ b/tests/functional/repl/add-overwritten-symbol-env.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. + +nix-repl> :a { a = "1"; } +Added 1 variables. +a + +nix-repl> :a { a = "2"; } +Added 1 variables. +a + +nix-repl> "result: ${a}" +"result: 2" diff --git a/tests/functional/repl/add-overwritten-symbol-env.in b/tests/functional/repl/add-overwritten-symbol-env.in new file mode 100644 index 000000000000..dbbf4967728d --- /dev/null +++ b/tests/functional/repl/add-overwritten-symbol-env.in @@ -0,0 +1,4 @@ +# COM: :a uses the newest version of a symbol +:a { a = "1"; } +:a { a = "2"; } +"result: ${a}" diff --git a/tests/functional/repl/add-variable-with-spaces.expected b/tests/functional/repl/add-variable-with-spaces.expected new file mode 100644 index 000000000000..dff5a0cbd2be --- /dev/null +++ b/tests/functional/repl/add-variable-with-spaces.expected @@ -0,0 +1,12 @@ +Nix +Type :? for help. + +nix-repl> :a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } +Added 4 variables. +a, b, longerName, "with spaces" + +nix-repl> :reload + +nix-repl> :l ./attribute-set.nix +Added 4 variables. +a, b, longerName, "with spaces" diff --git a/tests/functional/repl/add-variable-with-spaces.in b/tests/functional/repl/add-variable-with-spaces.in new file mode 100644 index 000000000000..763a61901248 --- /dev/null +++ b/tests/functional/repl/add-variable-with-spaces.in @@ -0,0 +1,3 @@ +:a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } +:reload +:l ./attribute-set.nix diff --git a/tests/functional/repl/attribute-set.nix b/tests/functional/repl/attribute-set.nix new file mode 100644 index 000000000000..7b2e71badd48 --- /dev/null +++ b/tests/functional/repl/attribute-set.nix @@ -0,0 +1,6 @@ +{ + a = 1; + b = 2; + longerName = 3; + "with spaces" = 4; +} diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index d2a5bf328535..29ca40cee2cc 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc curriedArgs Function `curriedArgs`\ diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 357cf9986808..65fbc2bd5d80 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 276de2e60b59..9fcf6169a5fc 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc compact Function `compact`\ diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index a68188b25abf..bda53bbfa660 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc constant error: value does not have documentation diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 3bf1c40715b1..e01f8868ee7b 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc floatedIn Function `floatedIn`\ diff --git a/tests/functional/repl/doc-functor.expected b/tests/functional/repl/doc-functor.expected index 8b86fe913448..e5984098c344 100644 --- a/tests/functional/repl/doc-functor.expected +++ b/tests/functional/repl/doc-functor.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-functor.nix -Added variables. +Added 13 variables. +diverging, doubler, helper, helper2, helper3, lib, makeOverridable, makeVeryOverridable, multiplier, multiply, recursive, recursive2, square nix-repl> :doc multiplier Function `__functor`\ diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 437c09d2b319..676ef73135dd 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc nonStrict Function `nonStrict`\ diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 862697613be6..ef176cea883f 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc measurement Function `measurement`\ diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index 21523e24c818..c2024b094918 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc multiply Function `multiply`\ diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 32ca9aef22ae..d2a452ba7bf3 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc unambiguous Function `unambiguous`\ diff --git a/tests/functional/repl/dollar-escaping.expected b/tests/functional/repl/dollar-escaping.expected new file mode 100644 index 000000000000..6e9d237f9162 --- /dev/null +++ b/tests/functional/repl/dollar-escaping.expected @@ -0,0 +1,5 @@ +Nix +Type :? for help. + +nix-repl> "$" + "{hi}" +"\${hi}" diff --git a/tests/functional/repl/dollar-escaping.in b/tests/functional/repl/dollar-escaping.in new file mode 100644 index 000000000000..39734f24b864 --- /dev/null +++ b/tests/functional/repl/dollar-escaping.in @@ -0,0 +1,2 @@ +# COM: Check dollar escaping https://github.com/NixOS/nix/issues/4909 +"$" + "{hi}" diff --git a/tests/functional/repl/file-a.nix b/tests/functional/repl/file-a.nix new file mode 100644 index 000000000000..f113eebfd9b9 --- /dev/null +++ b/tests/functional/repl/file-a.nix @@ -0,0 +1 @@ +{ fromA = 1; } diff --git a/tests/functional/repl/file-b.nix b/tests/functional/repl/file-b.nix new file mode 100644 index 000000000000..ddde63c16c42 --- /dev/null +++ b/tests/functional/repl/file-b.nix @@ -0,0 +1 @@ +{ fromB = 2; } diff --git a/tests/functional/repl/inherit-and-assignment.expected b/tests/functional/repl/inherit-and-assignment.expected new file mode 100644 index 000000000000..43ad52b5ef88 --- /dev/null +++ b/tests/functional/repl/inherit-and-assignment.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> x = { p = 10; } + +nix-repl> inherit (x) p; q = 20; + +nix-repl> p + q +30 diff --git a/tests/functional/repl/inherit-and-assignment.in b/tests/functional/repl/inherit-and-assignment.in new file mode 100644 index 000000000000..583b9096b5e8 --- /dev/null +++ b/tests/functional/repl/inherit-and-assignment.in @@ -0,0 +1,4 @@ +# COM: mixed bindings: inherit and assignment +x = { p = 10; } +inherit (x) p; q = 20; +p + q diff --git a/tests/functional/repl/inherit-current-scope.expected b/tests/functional/repl/inherit-current-scope.expected new file mode 100644 index 000000000000..d2b2f39d224a --- /dev/null +++ b/tests/functional/repl/inherit-current-scope.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> foo = 42 + +nix-repl> inherit foo + +nix-repl> foo +42 diff --git a/tests/functional/repl/inherit-current-scope.in b/tests/functional/repl/inherit-current-scope.in new file mode 100644 index 000000000000..d7eb0047afb2 --- /dev/null +++ b/tests/functional/repl/inherit-current-scope.in @@ -0,0 +1,4 @@ +# COM: inherit from current scope +foo = 42 +inherit foo +foo diff --git a/tests/functional/repl/inherit-missing-shows-pos.expected b/tests/functional/repl/inherit-missing-shows-pos.expected new file mode 100644 index 000000000000..dc02de2070f1 --- /dev/null +++ b/tests/functional/repl/inherit-missing-shows-pos.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. + +nix-repl> a = { x = 1; } + +nix-repl> inherit (a) y + +nix-repl> y +error: attribute 'y' missing + at «string»:1:13: + 1| inherit (a) y + | ^ + Did you mean x? diff --git a/tests/functional/repl/inherit-missing-shows-pos.in b/tests/functional/repl/inherit-missing-shows-pos.in new file mode 100644 index 000000000000..057b2e816e92 --- /dev/null +++ b/tests/functional/repl/inherit-missing-shows-pos.in @@ -0,0 +1,4 @@ +# COM: inherit error shows position (without spurious semicolon from retry) +a = { x = 1; } +inherit (a) y +y diff --git a/tests/functional/repl/inherit-multiple-attrs.expected b/tests/functional/repl/inherit-multiple-attrs.expected new file mode 100644 index 000000000000..e60df510a3cf --- /dev/null +++ b/tests/functional/repl/inherit-multiple-attrs.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { x = 10; y = 20; } + +nix-repl> inherit (a) x y + +nix-repl> x + y +30 diff --git a/tests/functional/repl/inherit-multiple-attrs.in b/tests/functional/repl/inherit-multiple-attrs.in new file mode 100644 index 000000000000..0da0815b2d15 --- /dev/null +++ b/tests/functional/repl/inherit-multiple-attrs.in @@ -0,0 +1,4 @@ +# COM: inherit multiple attributes +a = { x = 10; y = 20; } +inherit (a) x y +x + y diff --git a/tests/functional/repl/inherit-with-semicolon.expected b/tests/functional/repl/inherit-with-semicolon.expected new file mode 100644 index 000000000000..37d4020104d5 --- /dev/null +++ b/tests/functional/repl/inherit-with-semicolon.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { z = 99; } + +nix-repl> inherit (a) z; + +nix-repl> z +99 diff --git a/tests/functional/repl/inherit-with-semicolon.in b/tests/functional/repl/inherit-with-semicolon.in new file mode 100644 index 000000000000..d7a4972a2b3f --- /dev/null +++ b/tests/functional/repl/inherit-with-semicolon.in @@ -0,0 +1,4 @@ +# COM: inherit with semicolon +a = { z = 99; } +inherit (a) z; +z diff --git a/tests/functional/repl/inherit.expected b/tests/functional/repl/inherit.expected new file mode 100644 index 000000000000..9fde67b8cf46 --- /dev/null +++ b/tests/functional/repl/inherit.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { b = 1; c = 2; } + +nix-repl> inherit (a) b + +nix-repl> b +1 diff --git a/tests/functional/repl/inherit.in b/tests/functional/repl/inherit.in new file mode 100644 index 000000000000..804bbbec17a8 --- /dev/null +++ b/tests/functional/repl/inherit.in @@ -0,0 +1,4 @@ +# COM: Test inherit statement support (issue #15053) +a = { b = 1; c = 2; } +inherit (a) b +b diff --git a/tests/functional/repl/list-loaded-nothing-loaded.expected b/tests/functional/repl/list-loaded-nothing-loaded.expected new file mode 100644 index 000000000000..1f9f97c984d2 --- /dev/null +++ b/tests/functional/repl/list-loaded-nothing-loaded.expected @@ -0,0 +1,5 @@ +Nix +Type :? for help. + +nix-repl> :ll +error: nothing has been loaded yet diff --git a/tests/functional/repl/list-loaded-nothing-loaded.in b/tests/functional/repl/list-loaded-nothing-loaded.in new file mode 100644 index 000000000000..f47ea2c888f9 --- /dev/null +++ b/tests/functional/repl/list-loaded-nothing-loaded.in @@ -0,0 +1 @@ +:ll diff --git a/tests/functional/repl/multiple-bindings-same-line.expected b/tests/functional/repl/multiple-bindings-same-line.expected new file mode 100644 index 000000000000..77062af5fe63 --- /dev/null +++ b/tests/functional/repl/multiple-bindings-same-line.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> a = 1; b = 2; + +nix-repl> a + b +3 diff --git a/tests/functional/repl/multiple-bindings-same-line.in b/tests/functional/repl/multiple-bindings-same-line.in new file mode 100644 index 000000000000..6dd1d62a0e43 --- /dev/null +++ b/tests/functional/repl/multiple-bindings-same-line.in @@ -0,0 +1,3 @@ +# COM: multiple bindings on one line +a = 1; b = 2; +a + b diff --git a/tests/functional/repl/nested-attr-path.expected b/tests/functional/repl/nested-attr-path.expected new file mode 100644 index 000000000000..59164fede4b9 --- /dev/null +++ b/tests/functional/repl/nested-attr-path.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> a.b.c = 1; + +nix-repl> a.b +{ c = 1; } diff --git a/tests/functional/repl/nested-attr-path.in b/tests/functional/repl/nested-attr-path.in new file mode 100644 index 000000000000..4862c3c34bf4 --- /dev/null +++ b/tests/functional/repl/nested-attr-path.in @@ -0,0 +1,3 @@ +# COM: nested attribute path +a.b.c = 1; +a.b diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index 311855dae363..83ce76435783 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l pretty-print-idempotent.nix -Added variables. +Added 4 variables. +oneDeep, oneDeepList, twoDeep, twoDeepList nix-repl> oneDeep { homepage = "https://example.com"; } diff --git a/tests/functional/repl/printing.expected b/tests/functional/repl/printing.expected new file mode 100644 index 000000000000..f111939eb5b3 --- /dev/null +++ b/tests/functional/repl/printing.expected @@ -0,0 +1,59 @@ +Nix +Type :? for help. + +nix-repl> { a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +{ + a = { ... }; + l = [ ... ]; + n = 1234; + s = "string"; + x = { ... }; +} + +nix-repl> [ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +[ + 42 + 1 + "thingy" + { ... } + [ ... ] +] + +nix-repl> let x = { y = { a = 1; }; inherit x; }; in x +{ + x = «repeated»; + y = { ... }; +} + +nix-repl> :p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +{ + a = { b = 2; }; + n = 1234; + s = "string"; + x = { + y = { + z = { + y = «repeated»; + }; + }; + }; +} + +nix-repl> :p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] +[ + 42 + 1 + "thingy" + { + a = 1; + b = { + a = 1; + b = «repeated»; + }; + } + [ + 1 + 2 + 3 + ] +] diff --git a/tests/functional/repl/printing.in b/tests/functional/repl/printing.in new file mode 100644 index 000000000000..f3ca2cc88fcc --- /dev/null +++ b/tests/functional/repl/printing.in @@ -0,0 +1,11 @@ +# COM: Test recursive printing and formatting +# COM: Normal output should print attributes in lexicographical order non-recursively +{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +# COM: Same for lists, but order is preserved +[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +# COM: Same for let expressions +let x = { y = { a = 1; }; inherit x; }; in x +# COM: The :p command should recursively print sets, but prevent infinite recursion +:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +# COM: Same for lists +:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] diff --git a/tests/functional/repl/reload-with-non-existent-file.expected b/tests/functional/repl/reload-with-non-existent-file.expected new file mode 100644 index 000000000000..e15be6e73085 --- /dev/null +++ b/tests/functional/repl/reload-with-non-existent-file.expected @@ -0,0 +1,24 @@ +Nix +Type :? for help. + +nix-repl> :l file-a.nix +Added 1 variables. +fromA + +nix-repl> :l ./does-not-exist.nix +error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist + +nix-repl> :l file-b.nix +Added 1 variables. +fromB + +nix-repl> :r +Loading "file-a.nix"... +Added 1 variables. +fromA +Loading "file-b.nix"... +Added 1 variables. +fromB + +nix-repl> fromA + fromB +3 diff --git a/tests/functional/repl/reload-with-non-existent-file.in b/tests/functional/repl/reload-with-non-existent-file.in new file mode 100644 index 000000000000..740b3be9a412 --- /dev/null +++ b/tests/functional/repl/reload-with-non-existent-file.in @@ -0,0 +1,5 @@ +:l file-a.nix +:l ./does-not-exist.nix +:l file-b.nix +:r +fromA + fromB From a953d61686126d5cd39e5430bbf89674e3662261 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 17:33:36 +0300 Subject: [PATCH 094/364] derivaton-builder: Reap recursive-nix daemon worker threads early Not reaping threads early on leads to resource exhaustion when handling a lot of connection (like in nix-ninja). Ideally this would all just be async coroutine code so that we didn't have to spawn threads to handle connections, but it's a long-term goal. Fixes ENOMEM errors when building nix via nix-ninja. --- src/libstore/unix/build/derivation-builder.cc | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 3e9ad5434754..9c0463791e70 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include "nix/util/strings.hh" #include "nix/util/signals.hh" @@ -226,10 +228,16 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ std::thread daemonThread; + struct DaemonWorkerState + { + std::thread thread; + ref done; + }; + /** * The daemon worker threads. */ - std::vector daemonWorkerThreads; + std::list daemonWorkerThreads; const StorePathSet & originalPaths() override { @@ -1201,19 +1209,41 @@ void DerivationBuilderImpl::startDaemon() debug("received daemon connection"); - auto workerThread = std::thread([store, remote{std::move(remote)}]() { + auto doneFlag = make_ref(); + + auto workerThread = std::thread([doneFlag, store, remote{std::move(remote)}]() { try { daemon::processConnection( store, FdSource(remote.get()), FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (const Interrupted &) { debug("interrupted daemon connection"); - } catch (SystemError &) { + } catch (...) { + /* Swallow all exceptions to avoid crashing the the process (exceptions that escape from the thread + * trigger std::terminate()). */ ignoreExceptionExceptInterrupt(); } + + doneFlag->test_and_set(std::memory_order_relaxed); }); - daemonWorkerThreads.push_back(std::move(workerThread)); + daemonWorkerThreads.push_back( + DaemonWorkerState{ + .thread = std::move(workerThread), + .done = std::move(doneFlag), + }); + + /* Prune threads eagerly to free up resources. Ideally we'd also limit the number of concurrent workers. */ + for (auto it = daemonWorkerThreads.begin(), end = daemonWorkerThreads.end(); it != end;) { + auto & state = *it; + auto & thread = state.thread; + if (state.done->test(std::memory_order_relaxed) && thread.joinable()) { + thread.join(); + it = daemonWorkerThreads.erase(it); + } else { + ++it; + } + } } debug("daemon shutting down"); @@ -1242,9 +1272,7 @@ void DerivationBuilderImpl::stopDaemon() if (daemonThread.joinable()) daemonThread.join(); - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) + for (auto & [thread, doneFlag] : daemonWorkerThreads) thread.join(); daemonWorkerThreads.clear(); From 2acb40b2aa1c272dff4eb698921554c7ef948d30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2026 14:51:32 +0200 Subject: [PATCH 095/364] Don't destroy _fileTransfer on shutdown This is responsible for a lot of crash reports in Sentry (presumably due to destructor ordering issues). Since there is no cleanup done by this class that we care about, just leak it. --- src/libstore/filetransfer.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6d97fb4e3f5d..a07c92fe0cb1 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1183,14 +1183,16 @@ ref makeCurlFileTransfer(const FileTransferSettings & settings return make_ref(settings); } +static auto * const _fileTransfer = new Sync>; + ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer->lock()); - if (fileTransfer->state_.lock()->isQuitting()) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) + *fileTransfer = makeCurlFileTransfer().get_ptr(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer(const FileTransferSettings & settings) From 8f40805cc63d9e55e5e8409bdf53baad91be1850 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Apr 2026 21:29:39 +0100 Subject: [PATCH 096/364] feat(libutil): add `memo(f0)` memoization combinator Memoizes a `fun`. Useful because "call by need" has its use cases outside the evaluator too. Motivating use case: libfetchers lazy attributes. --- src/libutil-tests/memo.cc | 46 ++++++++++++++++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/include/nix/util/memo.hh | 40 +++++++++++++++++++++ src/libutil/include/nix/util/meson.build | 1 + 4 files changed, 88 insertions(+) create mode 100644 src/libutil-tests/memo.cc create mode 100644 src/libutil/include/nix/util/memo.hh diff --git a/src/libutil-tests/memo.cc b/src/libutil-tests/memo.cc new file mode 100644 index 000000000000..76be65260953 --- /dev/null +++ b/src/libutil-tests/memo.cc @@ -0,0 +1,46 @@ +#include +#include + +#include "nix/util/memo.hh" + +namespace nix { + +TEST(memo, computesOnce) +{ + int calls = 0; + fun f = memo([&calls]() -> int { + calls++; + return 42; + }); + EXPECT_EQ(f(), 42); + EXPECT_EQ(f(), 42); + EXPECT_EQ(f(), 42); + EXPECT_EQ(calls, 1); +} + +TEST(memo, copiesShareCache) +{ + int calls = 0; + fun f = memo([&calls]() -> int { + calls++; + return 7; + }); + auto g = f; + EXPECT_EQ(f(), 7); + EXPECT_EQ(g(), 7); + EXPECT_EQ(calls, 1); +} + +TEST(memo, worksWithString) +{ + int calls = 0; + fun f = memo([&calls]() -> std::string { + calls++; + return "hello"; + }); + EXPECT_EQ(f(), "hello"); + EXPECT_EQ(f(), "hello"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 6a86504ded42..527d5e3fbff5 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -79,6 +79,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memo.cc', 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nar-listing.cc', diff --git a/src/libutil/include/nix/util/memo.hh b/src/libutil/include/nix/util/memo.hh new file mode 100644 index 000000000000..ba09bac6eb75 --- /dev/null +++ b/src/libutil/include/nix/util/memo.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include "nix/util/fun.hh" + +#include +#include +#include + +namespace nix { + +/** + * Memoize a `fun`. + * + * Copies of the returned `fun` share the same cache. + * Thread-safe. + */ +template +fun memo(fun f) +{ + struct State + { + fun compute; + std::once_flag flag; + std::optional cached; + + explicit State(fun compute) + : compute(std::move(compute)) + { + } + }; + + auto state = std::shared_ptr(new State(std::move(f))); + return [state]() -> T { + std::call_once(state->flag, [&]() { state->cached.emplace(state->compute()); }); + return *state->cached; + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 0f7a40df7a8c..cea6d48f5920 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -60,6 +60,7 @@ headers = [ config_pub_h ] + files( 'json-utils.hh', 'logging.hh', 'lru-cache.hh', + 'memo.hh', 'memory-source-accessor.hh', 'mounted-source-accessor.hh', 'muxable-pipe.hh', From ee78fe13ecc2e7d49d4763661760960b4c949362 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 21 Apr 2026 12:48:48 +0100 Subject: [PATCH 097/364] feat(libfetchers): add lazy attribute values Extend the Attr variant with a LazyAttr alternative: a deferred computation that is only evaluated when the attribute value is needed. This lets fetchers defer expensive work (like revCount) until an expression demands it. The existing getters and serialization transparently force any lazy attrs. --- src/libfetchers-tests/attrs.cc | 75 +++++++++++++++++++ src/libfetchers-tests/meson.build | 1 + src/libfetchers/attrs.cc | 35 ++++++--- src/libfetchers/include/nix/fetchers/attrs.hh | 26 ++++++- src/libflake/flake-primops.cc | 3 +- 5 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 src/libfetchers-tests/attrs.cc diff --git a/src/libfetchers-tests/attrs.cc b/src/libfetchers-tests/attrs.cc new file mode 100644 index 000000000000..4d0cbbb9ef74 --- /dev/null +++ b/src/libfetchers-tests/attrs.cc @@ -0,0 +1,75 @@ +#include + +#include "nix/fetchers/attrs.hh" + +#include + +namespace nix::fetchers { + +TEST(LazyAttr, resolveToInt) +{ + Attrs attrs; + attrs.insert_or_assign( + "count", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(42); + }}))); + EXPECT_EQ(maybeGetIntAttr(attrs, "count"), 42); +} + +TEST(LazyAttr, resolveToString) +{ + Attrs attrs; + attrs.insert_or_assign( + "name", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("hello"); + }}))); + EXPECT_EQ(maybeGetStrAttr(attrs, "name"), "hello"); +} + +TEST(LazyAttr, resolveToBool) +{ + Attrs attrs; + attrs.insert_or_assign( + "flag", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return Explicit{true}; + }}))); + EXPECT_EQ(maybeGetBoolAttr(attrs, "flag"), true); +} + +TEST(LazyAttr, attrsToJSONForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "x", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(99); + }}))); + auto json = attrsToJSON(attrs); + EXPECT_EQ(json["x"], 99); +} + +TEST(LazyAttr, attrsToQueryForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "v", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("val"); + }}))); + auto query = attrsToQuery(attrs); + EXPECT_EQ(query.at("v"), "val"); +} + +TEST(LazyAttr, notCalledUntilForced) +{ + int calls = 0; + Attrs attrs; + attrs.insert_or_assign( + "lazy", LazyAttr(make_ref(LazyAttrComputation{.compute = [&calls]() -> ResolvedAttr { + calls++; + return uint64_t(1); + }}))); + EXPECT_EQ(calls, 0); + maybeGetIntAttr(attrs, "lazy"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956b9..8cbb42e21375 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -40,6 +40,7 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', + 'attrs.cc', 'git-utils.cc', 'git.cc', 'input.cc', diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index cc9e72af460d..39e7a0fd5d6c 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -4,6 +4,18 @@ namespace nix::fetchers { +ResolvedAttr forceAttr(const Attr & attr) +{ + return std::visit( + overloaded{ + [](const LazyAttr & lazy) -> ResolvedAttr { return lazy->compute(); }, + [](const std::string & v) -> ResolvedAttr { return v; }, + [](uint64_t v) -> ResolvedAttr { return v; }, + [](const Explicit & v) -> ResolvedAttr { return v; }, + }, + attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; @@ -26,11 +38,12 @@ nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { json[attr.first] = v->t; } else unreachable(); @@ -43,7 +56,8 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } @@ -61,7 +75,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not an integer", name); } @@ -79,7 +94,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if>(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if>(&resolved)) return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -96,11 +112,12 @@ StringMap attrsToQuery(const Attrs & attrs) { StringMap query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, fmt("%d", *v)); - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, *v); - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); } else unreachable(); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 8a21b8ddbf69..60730d65d32b 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/hash.hh" +#include "nix/util/ref.hh" +#include "nix/util/fun.hh" #include @@ -12,7 +14,24 @@ namespace nix::fetchers { -typedef std::variant> Attr; +/** + * The resolved (non-lazy) subset of attribute value types. + */ +using ResolvedAttr = std::variant>; + +/** + * A deferred attribute computation. Wrapping in `ref<>` gives + * pointer-identity equality/ordering, which is correct: two lazy + * attrs are equal iff they are the same computation. + */ +struct LazyAttrComputation +{ + fun compute; +}; + +using LazyAttr = ref; + +using Attr = std::variant, LazyAttr>; /** * An `Attrs` can be thought of a JSON object restricted or simplified @@ -21,6 +40,11 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * Force a potentially lazy attribute to its resolved value. + */ +ResolvedAttr forceAttr(const Attr & attr); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 66962a014879..5ee587c3e051 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -105,12 +105,13 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); auto & vv = binds.alloc(s); + auto resolved = forceAttr(value); std::visit( overloaded{ [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const Explicit & value) { vv.mkBool(value.t); }}, - value); + resolved); } v.mkAttrs(binds); } From a08c14bf8b8dd91034a7fe7c1014b05aee12d772 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 21 Apr 2026 13:56:25 +0100 Subject: [PATCH 098/364] feat(libexpr): emit thunks for lazy fetcher attributes The "revCount" attribute is now emitted as a lazy attribute if libfetchers were to return it as such; as will be done in next commit. --- src/libexpr-tests/lazy-fetcher-attr.cc | 100 ++++++++++++++++++ src/libexpr-tests/meson.build | 1 + src/libexpr/include/nix/expr/fetch-tree.hh | 18 ++++ src/libexpr/include/nix/expr/meson.build | 1 + src/libexpr/primops/fetchTree.cc | 95 ++++++++++++++++- src/libfetchers/attrs.cc | 10 ++ src/libfetchers/include/nix/fetchers/attrs.hh | 5 + src/libflake/flake.cc | 1 + src/libflake/include/nix/flake/flake.hh | 8 -- 9 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 src/libexpr-tests/lazy-fetcher-attr.cc create mode 100644 src/libexpr/include/nix/expr/fetch-tree.hh diff --git a/src/libexpr-tests/lazy-fetcher-attr.cc b/src/libexpr-tests/lazy-fetcher-attr.cc new file mode 100644 index 000000000000..4c36424ecb55 --- /dev/null +++ b/src/libexpr-tests/lazy-fetcher-attr.cc @@ -0,0 +1,100 @@ +#include + +#include "nix/expr/fetch-tree.hh" +#include "nix/expr/tests/libexpr.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/store/path.hh" + +namespace nix { + +class LazyFetcherAttrTest : public LibExprTest +{ +protected: + StorePath dummyPath() + { + return StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-test"}; + } +}; + +TEST_F(LazyFetcherAttrTest, nonLazyAttrProducesImmediateValue) +{ + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("revCount", uint64_t(5)); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 5); +} + +TEST_F(LazyFetcherAttrTest, lazyAttrProducesThunk) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(42); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + + // Not yet forced, so the lazy function should not have been called + EXPECT_EQ(calls, 0); + + // Force the thunk + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 42); + EXPECT_EQ(calls, 1); +} + +TEST_F(LazyFetcherAttrTest, lazyFunctionOnlyCalledOnAccess) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("lastModified", uint64_t(1000)); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(99); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + // Access lastModified, so should not trigger lazy revCount + auto * lmAttr = v.attrs()->get(state.symbols.create("lastModified")); + ASSERT_NE(lmAttr, nullptr); + state.forceValue(*lmAttr->value, noPos); + EXPECT_EQ(lmAttr->value->integer().value, 1000); + EXPECT_EQ(calls, 0); + + // Now access revCount + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 99); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c20654..d18d5a4830d5 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -51,6 +51,7 @@ sources = files( 'error_traces.cc', 'eval.cc', 'json.cc', + 'lazy-fetcher-attr.cc', 'main.cc', 'nix_api_expr.cc', 'nix_api_external.cc', diff --git a/src/libexpr/include/nix/expr/fetch-tree.hh b/src/libexpr/include/nix/expr/fetch-tree.hh new file mode 100644 index 000000000000..3eb8a01c0c5f --- /dev/null +++ b/src/libexpr/include/nix/expr/fetch-tree.hh @@ -0,0 +1,18 @@ +#pragma once + +#include "nix/expr/eval.hh" + +namespace nix { + +/** + * Convert a libfetchers `Input` to libexpr `Value`. + */ +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 4213476fe73a..7334b42f8f17 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,6 +20,7 @@ headers = [ config_pub_h ] + files( 'eval-profiler.hh', 'eval-settings.hh', 'eval.hh', + 'fetch-tree.hh', 'function-trace.hh', 'gc-small-vector.hh', 'get-drvs.hh', diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index afd61e90fbb5..429009301db6 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,7 +1,9 @@ +#include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" @@ -19,6 +21,95 @@ namespace nix { +/** + * Adapter for putting libfetchers data into a thunk closure. + * Used as the argument to prim_forceLazyFetcherAttr in a lazy apply thunk. + */ +class LazyFetcherAttr : public ExternalValueBase, public gc_cleanup +{ + fetchers::LazyAttr lazy; + +public: + LazyFetcherAttr(fetchers::LazyAttr lazy) + : lazy(std::move(lazy)) + { + } + + fetchers::ResolvedAttr force() + { + return lazy->compute(); + } + +protected: + std::ostream & print(std::ostream & str) const override + { + unreachable(); + } + +public: + std::string showType() const override + { + unreachable(); + } + + std::string typeOf() const override + { + unreachable(); + } +}; + +/** + * Initialize a `Value` from a resolved fetcher attribute. + */ +static void resolvedAttrToValue(EvalState & state, Value & v, const fetchers::ResolvedAttr & resolved) +{ + std::visit( + overloaded{ + [&](const std::string & s) { v.mkString(s, state.mem); }, + [&](uint64_t n) { v.mkInt(n); }, + [&](const Explicit & b) { v.mkBool(b.t); }, + }, + resolved); +} + +/** + * internal primop: Force a LazyFetcherAttr external value. + */ +static void prim_forceLazyFetcherAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + Value & arg = *args[0]; + + state.forceValue(arg, pos); + // We only construct this primop with LazyFetcherAttr preapplied. + assert(arg.type() == nExternal); + auto * ext = dynamic_cast(args[0]->external()); + assert(ext); + + resolvedAttrToValue(state, v, ext->force()); +} + +/** + * Emit a lazy thunk for a LazyAttr: mkApp(primop, externalValue). + */ +static void emitLazyAttrThunk(EvalState & state, const fetchers::LazyAttr & lazyAttr, Value & dest) +{ + // not user-callable (unregistered, internal) + static PrimOp forcePrimOp{ + .name = "__forceLazyFetcherAttr", + .arity = 1, + .impl = prim_forceLazyFetcherAttr, + .internal = true, + }; + + auto * vExt = state.allocValue(); + vExt->mkExternal(new LazyFetcherAttr(lazyAttr)); + + auto * vPrimOp = state.allocValue(); + vPrimOp->mkPrimOp(&forcePrimOp); + + dest.mkApp(vPrimOp, vExt); +} + void emitTreeAttrs( EvalState & state, const StorePath & storePath, @@ -51,7 +142,9 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); } - if (auto revCount = input.getRevCount()) + if (auto revCount = maybeGetLazyAttr(input.attrs, "revCount")) + emitLazyAttrThunk(state, *revCount, attrs.alloc("revCount")); + else if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) attrs.alloc("revCount").mkInt(0); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 39e7a0fd5d6c..f3ecc3a87bc0 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -51,6 +51,16 @@ nlohmann::json attrsToJSON(const Attrs & attrs) return json; } +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) + return {}; + if (auto v = std::get_if(&i->second)) + return *v; + return {}; +} + std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 60730d65d32b..8eede58086e5 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -45,6 +45,11 @@ typedef std::map Attrs; */ ResolvedAttr forceAttr(const Attr & attr); +/** + * Retrieve an attr, but only if it's a LazyAttr. + */ +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index e8dbf42f5451..03e68e1f995a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -38,6 +38,7 @@ #include "nix/fetchers/input-cache.hh" #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fd52dbebac5d..aa063a08e396 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -233,14 +233,6 @@ ref openEvalCache(EvalState & state, ref Date: Tue, 21 Apr 2026 14:08:14 +0100 Subject: [PATCH 099/364] feat(git): make revCount lazy revCount requires an expensive walk of the entire commit graph. Now it's wrapped in a LazyAttr so the cost is only paid when the Nix expression actually accesses `.revCount`. This also fixes fetching from shallow clones with fetchGit. Previously the eager revCount computation failed the whole fetch, but now `.outPath` and some other attributes succeed, while `.revCount` access fails (as expected for shallow repos). --- src/libfetchers/git.cc | 46 +++++++++++++++++++---------- tests/functional/fetchGitShallow.sh | 8 +++-- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 447b4a3694f7..6baf7d2e525c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -13,6 +13,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/memo.hh" #include "nix/util/mounted-source-accessor.hh" #include @@ -163,6 +164,13 @@ std::vector getPublicKeys(const Attrs & attrs) static const Hash nullRev{HashAlgorithm::SHA1}; +static LazyAttr makeLazyAttr(fun compute) +{ + return make_ref(LazyAttrComputation{ + .compute = memo(std::move(compute)), + }); +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override @@ -723,14 +731,12 @@ struct GitInputScheme : InputScheme } uint64_t getRevCount( - const Settings & settings, - const RepoInfo & repoInfo, - const std::filesystem::path & repoDir, - const Hash & rev) const + ref cache, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { - Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + if (GitRepo::openRepo(repoDir, {})->isShallow()) + throw Error("'%s' is a shallow Git repository, so 'revCount' is not available", repoInfo.locationToArg()); - auto cache = settings.getCache(); + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -745,6 +751,18 @@ struct GitInputScheme : InputScheme return revCount; } + LazyAttr lazyRevCount( + const Settings & settings, + const RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev) const + { + auto cache = settings.getCache(); + return makeLazyAttr([this, cache, repoInfo, repoDir, rev]() -> ResolvedAttr { + return getRevCount(cache, repoInfo, repoDir, rev); + }); + } + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( @@ -891,13 +909,6 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(repoDir, {}); - auto isShallow = repo->isShallow(); - - if (isShallow && !getShallowAttr(input)) - throw Error( - "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", - repoInfo.locationToArg()); - // FIXME: check whether rev is an ancestor of ref? auto rev = *input.getRev(); @@ -911,7 +922,7 @@ struct GitInputScheme : InputScheme if (!getShallowAttr(input)) { /* Like lastModified, skip revCount if supplied by the caller. */ if (!input.attrs.contains("revCount")) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoDir, rev)); } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -1033,8 +1044,11 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); if (!getShallowAttr(input)) { - input.attrs.insert_or_assign( - "revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); + if (rev == nullRev) { + input.attrs.insert_or_assign("revCount", uint64_t(0)); + } else { + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoPath, rev)); + } } verifyCommit(input, repo); diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 6b91d60cd9e3..67a8e3d3bd16 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -29,9 +29,11 @@ git -C "$TEST_ROOT/shallow-parent" commit -m "Branch commit" # Make a shallow clone (depth=1) git clone --depth 1 "file://$TEST_ROOT/shallow-parent" "$TEST_ROOT/shallow-clone" -# Test 1: Fetching a shallow repo shouldn't work by default, because we can't -# return a revCount. -(! nix eval --impure --raw --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).outPath") +# Test 1: Fetching a shallow repo succeeds for outPath because revCount is lazy. +path1=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).outPath") +[[ -d "$path1" ]] +# But accessing revCount on a shallow clone fails. +(! nix eval --impure --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).revCount" 2>/dev/null) # Test 2: But you can request a shallow clone, which won't return a revCount. path=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow-clone\"; ref = \"dev\"; shallow = true; }).outPath") From 0c7b61dadd1b575bb8abc36e66a53e339988d200 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 30 Apr 2026 02:07:29 +0200 Subject: [PATCH 100/364] fix(flake): don't force revCount in fingerprint The flake eval cache fingerprint eagerly forced revCount via getRevCount(), which defeats lazy revCount on shallow clones. Since revCount is functionally determined by rev (already part of the fingerprint), we only need to include its *presence* in the fingerprint, not its value. Note that before these changes, Nix had a bug where it could produce a wrong revcount on shallow clones. It arguably should have inferred `shallow = true;` but it didn't and I'm not changing that behavior either. Leaving it at its default is more "pure" in a sense, but a case could certainly be made to let the CLI infer the value that works automagically. (Purity in the CLI is a spectrum; system attribute selection could be argued to be impure too, for instance.) --- src/libflake/flake.cc | 10 +++++++--- tests/functional/fetchGitShallow.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 03e68e1f995a..c34fa2d0a1ba 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -991,9 +991,13 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc /* Include revCount and lastModified because they're not necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. */ - if (auto revCount = flake.lockedRef.input.getRevCount()) - *fingerprint += fmt(";revCount=%d", *revCount); + tarball flakes) but can influence the evaluation result. + For revCount, we only include its presence (not its value) + because the value is functionally determined by rev, which + is already part of the fingerprint. This avoids forcing a + lazy revCount computation. */ + if (flake.lockedRef.input.attrs.contains("revCount")) + *fingerprint += ";hasRevCount"; if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 67a8e3d3bd16..0da36ee9bac9 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -65,3 +65,32 @@ fi # Verify that we can shallow fetch the worktree git -C "$TEST_ROOT/shallow-worktree" rev-list --count HEAD >/dev/null nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$TEST_ROOT/shallow-worktree\"; shallow = true; }).rev" + +# Test 5: nix build --dry-run on a shallow clone must not force revCount. +# The flake fingerprint must not eagerly evaluate revCount, because that +# would fail (or produce wrong results) on shallow clones. +# Nor should it generate a complete lock file that serializes the root node. +# We remove the parent repo to ensure that a future improvement that +# tries to fetch missing history can't paper over the issue. +createGitRepo "$TEST_ROOT/shallow-build-parent" +echo "" > "$TEST_ROOT/shallow-build-parent/file.txt" +git -C "$TEST_ROOT/shallow-build-parent" add file.txt +git -C "$TEST_ROOT/shallow-build-parent" commit -m "first" +cat > "$TEST_ROOT/shallow-build-parent/flake.nix" < \\\$out" ]; + }; + }; +} +EOF +git -C "$TEST_ROOT/shallow-build-parent" add flake.nix +git -C "$TEST_ROOT/shallow-build-parent" commit -m "add flake" +git clone --depth 1 "file://$TEST_ROOT/shallow-build-parent" "$TEST_ROOT/shallow-build-clone" +rm -rf "$TEST_ROOT/shallow-build-parent" +nix build --dry-run "git+file://$TEST_ROOT/shallow-build-clone" From ad649e34a3e630c3a91bab5217e422439edcee08 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 May 2026 00:29:59 +0200 Subject: [PATCH 101/364] chore: remove redundant comment --- src/libutil/include/nix/util/fun.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/include/nix/util/fun.hh b/src/libutil/include/nix/util/fun.hh index c480ffe71711..59a7cc87939c 100644 --- a/src/libutil/include/nix/util/fun.hh +++ b/src/libutil/include/nix/util/fun.hh @@ -1,6 +1,5 @@ #pragma once ///@file -// Tests in: src/libutil-tests/fun.cc #include #include From 7609abe60a6359dc349cf8ab645c910a5eae0293 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 13:11:19 +0200 Subject: [PATCH 102/364] Document how to use GC roots safely --- src/libstore/include/nix/store/store-api.hh | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index bfd4ffce2181..b6114cce65e7 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -816,6 +816,47 @@ public: /** * Add a store path as a temporary root of the garbage collector. * The root disappears as soon as we exit. + * Before exiting, if you want to avoid the path being GC'ed, you either have to make it a permanent root using + * `LocalFSStore::addPermRoot()`, or make sure it's reachable from a permanent root (e.g. by adding it as a + * reference of a reachable path). + * + * To avoid races, you should call either this function or `LocalFSStore::addPermRoot()` *before* creating and using + * a store path, e.g. + * + * ```c++ + * auto path = store.computeStorePath(...); + * store->addTempRoot(path); + * if (!store->isValidPath(path)) + * store->addToStore(...); + * ``` + * + * By contrast, registering a root just before *using* a path is not sufficient to prevent GC races. For + * instance, don't do this: + * + * ```c++ + * store->addTempRoot(path); + * auto drv = store->readDerivation(path); + * ``` + * + * since the path may be GC'ed just before the call to `addTempRoot()`. + * + * Note that `addToStore()` implicitly calls `addTempRoot()`, so you don't need to call it yourself if you're + * calling `addToStore()` unconditionally. + * + * It is generally the responsibility of the caller of Nix APIs and CLI tools to ensure that paths are reachable by + * the garbage collector. For example, `buildPath(drvPath)` does not need to register *drvPath* as a GC root, since + * that's the responsibility of the caller, and it would be too late for `buildPath()` to do so anyway. Thus, this + * can race: + * ```console + * drv=$(nix-instantiate foo.nix) + * nix-store -r $drv + * ``` + * whereas this is safe: + * ```console + * nix-instantiate foo.nix --add-root ./drv + * nix-store -r ./drv + * ``` + * */ virtual void addTempRoot(const StorePath & path) { From b87d9e810b759a703c7d72c0cbe04414b2edcf22 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 13:11:38 +0200 Subject: [PATCH 103/364] Add TODO item --- src/libfetchers/fetch-to-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 3af2d4c83e88..09112a965584 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -82,6 +82,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun ? [&]() { + // FIXME: we may have already computed this above. auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( From e2490c72941e6eb34af2455e3bc459644be55755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:42:50 +0200 Subject: [PATCH 104/364] StoreDirConfig::parseStorePath(): Don't crash on empty paths --- src/libstore/store-dir-config.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 61f82029d79f..962a3830f166 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -8,6 +8,8 @@ namespace nix { StorePath StoreDirConfig::parseStorePath(std::string_view path) const { + if (path.empty()) + throw BadStorePath("empty path is not a valid store path"); // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native // notion of a canonical path at all. For example, it makes to From 8f3d7023cb4c168fa45dedac120efdb46794ed10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:43:01 +0200 Subject: [PATCH 105/364] canonPath(): Don't crash on empty paths --- src/libutil-tests/file-system.cc | 2 +- src/libutil/file-system.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index f290e1178090..417622a47cf2 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -114,7 +114,7 @@ TEST(canonPath, requiresAbsolutePath) ASSERT_ANY_THROW(canonPath("."sv)); ASSERT_ANY_THROW(canonPath(".."sv)); ASSERT_ANY_THROW(canonPath("../"sv)); - ASSERT_DEATH({ canonPath(""sv); }, "!path.empty\\(\\)"); + ASSERT_ANY_THROW(canonPath(""sv)); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b3087700d733..b6ad1e128046 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -95,7 +95,8 @@ absPath(const std::filesystem::path & path0, const std::filesystem::path * dir, std::filesystem::path canonPath(const std::filesystem::path & path, bool resolveSymlinks) { - assert(!path.empty()); + if (path.empty()) + throw Error("cannot canonicalise an empty path"); if (!path.is_absolute()) throw Error("not an absolute path: %s", PathFmt(path)); From 1a4d5782ffd33f6069295b3b1488bb402e34a9c2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 6 May 2026 03:02:45 +0300 Subject: [PATCH 106/364] Clean up and deduplicate some Logger code Puts some logger classes in anonymous namespaces. This also appeases Wweak-vtables and avoids making those symbols visible in the DSO. Also marks those classes as final and deduplicates storePathToNameWithoutDrvSuffix in the progress bar code. --- src/libmain/progress-bar.cc | 35 ++++++++++++++++++++++------------- src/libutil/tee-logger.cc | 7 ++++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a8044a240d51..f81e90a06e7a 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -2,8 +2,10 @@ #include "nix/util/terminal.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/path.hh" +#include "nix/util/file-system.hh" #include "nix/store/names.hh" +#include "nix/util/util.hh" #include #include @@ -13,17 +15,19 @@ namespace nix { +namespace { + static std::string_view getS(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + throw Error("could not get expected log field of type 'string' at index %d", n); return fields[n].s; } static uint64_t getI(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + throw Error("could not get expected log field of type 'int' at index %d", n); return fields[n].i; } @@ -34,10 +38,17 @@ static std::string_view storePathToName(std::string_view path) return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); } -class ProgressBar : public Logger +static std::string_view storePathToNameWithoutDrvSuffix(std::string_view path) { -private: + auto res = storePathToName(path); + if (hasSuffix(res, drvExtension)) + res.remove_suffix(drvExtension.size()); + return res; +} +class ProgressBar final : public Logger +{ +private: struct ActInfo { std::string s, lastLine, phase; @@ -223,9 +234,7 @@ class ProgressBar : public Logger state->activitiesByType[type].its.emplace(act, i); if (type == actBuild) { - std::string name(storePathToName(getS(fields, 0))); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") @@ -250,9 +259,7 @@ class ProgressBar : public Logger } if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } @@ -687,6 +694,8 @@ class ProgressBar : public Logger } }; +} // namespace + std::unique_ptr makeProgressBar() { return std::make_unique(isTTY()); diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 8433168a5a82..8e27714e3f40 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -2,10 +2,13 @@ namespace nix { -struct TeeLogger : Logger +namespace { + +class TeeLogger final : public Logger { std::vector> loggers; +public: TeeLogger(std::vector> && loggers) : loggers(std::move(loggers)) { @@ -94,6 +97,8 @@ struct TeeLogger : Logger } }; +} // namespace + std::unique_ptr makeTeeLogger(std::unique_ptr mainLogger, std::vector> && extraLoggers) { From a253942be956f946744c3da3f4e694934b56cc51 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 6 May 2026 03:07:07 +0300 Subject: [PATCH 107/364] libmain: Hide/unhide cursor in the progress bar Uses corresponding ANSI/VT escape sequences to hide/unhide the cursor when the output is a TTY. nom does the same AFAICT. --- src/libmain/progress-bar.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f81e90a06e7a..512ba4faad67 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -107,6 +107,18 @@ class ProgressBar final : public Logger std::unique_ptr interruptCallback; + void hideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25l"); + } + + void unhideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25h"); + } + public: ProgressBar(bool isTTY) @@ -116,6 +128,7 @@ class ProgressBar final : public Logger redraw("\rshutting down\e[K"); })) { + hideCursorIfNeeded(); state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); @@ -142,6 +155,7 @@ class ProgressBar final : public Logger if (state->active) { state->active = false; clearProgressDisplay(); + unhideCursorIfNeeded(); updateCV.notify_one(); quitCV.notify_one(); } @@ -159,8 +173,10 @@ class ProgressBar final : public Logger return; } - if (state->active) + if (state->active) { clearProgressDisplay(); + unhideCursorIfNeeded(); + } } void resume() override @@ -173,8 +189,10 @@ class ProgressBar final : public Logger state->suspensions--; } if (state->suspensions == 0) { - if (state->active) + if (state->active) { clearProgressDisplay(); + hideCursorIfNeeded(); + } state->haveUpdate = true; updateCV.notify_one(); } @@ -681,7 +699,9 @@ class ProgressBar final : public Logger return {}; invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); + unhideCursorIfNeeded(); auto s = trim(readLine(getStandardInput(), true)); + hideCursorIfNeeded(); if (s.size() != 1) return {}; draw(*state); From 4fa3a8cad2b651ab0bb3a30d78d9499c8ae16216 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 7 May 2026 00:41:57 +0300 Subject: [PATCH 108/364] libstore: Use member fileTransfer instead of global getFileTransfer() in the s3 store Ideally we'd get rid of the global file transfer object altogether and this moves us in that direction. fileTransfer was already a member of the HttpBinaryCacheStore, so just use that instead. In practice it pointed to the same singleton object though (it's primarily used for tests now). --- src/libstore/s3-binary-cache-store.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f6d300c0936f..54bf1d0ad7c3 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -322,7 +322,7 @@ std::string S3BinaryCacheStore::createMultipartUpload( std::move(headers->begin(), headers->end(), std::back_inserter(req.headers)); } - auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(req).get(); std::regex uploadIdRegex("([^<]+)"); std::smatch match; @@ -353,7 +353,7 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, req.data = {payload}; req.mimeType = "application/octet-stream"; - auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(req).get(); if (result.etag.empty()) { throw Error("S3 UploadPart response missing ETag for part %d", partNumber); @@ -374,7 +374,7 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_ req.uri = VerbatimURL(url); req.method = HttpMethod::Delete; - getFileTransfer()->enqueueFileTransfer(req).get(); + fileTransfer->enqueueFileTransfer(req).get(); } catch (...) { ignoreExceptionInDestructor(); } @@ -407,7 +407,7 @@ void S3BinaryCacheStore::completeMultipartUpload( req.data = {payload}; req.mimeType = "text/xml"; - getFileTransfer()->enqueueFileTransfer(req).get(); + fileTransfer->enqueueFileTransfer(req).get(); debug("S3 multipart upload completed: %d parts uploaded for '%s'", partEtags.size(), key); } From 220ccc746d156b5ea62c45b42e91a4b6f380f6ef Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 May 2026 01:26:05 +0200 Subject: [PATCH 109/364] fix: Restore fingerprint value for untrustworthy revCount values It was fine for trustworthy revCounts, but technically the functional dependency described does not always hold up, or we can not trust it to. - buggy tarball providers - merge conflicts in lock files or other kinds of chaos in "user space" --- src/libflake/flake.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index c34fa2d0a1ba..bd3ea4acbc65 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -989,15 +989,24 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); - /* Include revCount and lastModified because they're not - necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. - For revCount, we only include its presence (not its value) - because the value is functionally determined by rev, which - is already part of the fingerprint. This avoids forcing a - lazy revCount computation. */ - if (flake.lockedRef.input.attrs.contains("revCount")) - *fingerprint += ";hasRevCount"; + if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { + if (std::get_if(revCount)) { + /* A lazy revCount is computed by the fetcher, so its + value is functionally determined by `rev`. We only + need to record its presence, not force its value. + + This means a lazy and a concrete revCount that would + resolve to the same value produce different + fingerprints, sacrificing some cache hits to avoid + the cost of forcing. */ + *fingerprint += ";hasRevCount"; + } else if (auto n = flake.lockedRef.input.getRevCount()) { + /* A concrete revCount comes from a lockfile or explicit + user input. The fetcher passes it through as-is, so + it can affect evaluation and must be fingerprinted. */ + *fingerprint += fmt(";revCount=%d", *n); + } + } if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); From 2a42376bbe96340cf5f94b96d881225b5366bf59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 8 May 2026 10:54:10 +0000 Subject: [PATCH 110/364] tests/nixos: serve fetchers-substitute cache via nginx nix-serve depends on the perl bindings, which are slated for removal, and nix-serve-ng adds an extra dependency just to serve a static binary cache over HTTP. Replace it with plain nginx serving a file:// binary cache populated via 'nix copy', which is simpler and only relies on components already in nixpkgs. :house: Remote-Dev: homespace --- tests/nixos/fetchers-substitute.nix | 83 +++++++++-------------------- 1 file changed, 24 insertions(+), 59 deletions(-) diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index b61a3929af80..0959512e1db6 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -11,22 +11,18 @@ "fetch-tree" ]; - networking.firewall.allowedTCPPorts = [ 5000 ]; + networking.firewall.allowedTCPPorts = [ 80 ]; - # TODO stop using this, because it has to depend on an older version of Nix that still has the perl bindings. - services.nix-serve = { + systemd.tmpfiles.rules = [ "d /var/cache/binary-cache 0755 root root -" ]; + + services.nginx = { enable = true; - secretKeyFile = - let - key = pkgs.writeTextFile { - name = "secret-key"; - text = '' - substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== - ''; - }; - in - "${key}"; + virtualHosts."substituter".root = "/var/cache/binary-cache"; }; + + environment.etc."nix/secret-key".text = '' + substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== + ''; }; nodes.importer = @@ -39,13 +35,12 @@ "nix-command" "fetch-tree" ]; - substituters = lib.mkForce [ "http://substituter:5000" ]; + substituters = lib.mkForce [ "http://substituter" ]; trusted-public-keys = lib.mkForce [ "substituter:EkFXxh3upwnPjUXg41d0HRWDzBoseBTINPiv0zNQd2g=" ]; }; }; - testScript = - { nodes }: # python + testScript = # python '' import json import os @@ -53,11 +48,11 @@ start_all() substituter.wait_for_unit("multi-user.target") + importer.wait_for_unit("multi-user.target") - ########################################## - # Test 1: builtins.fetchurl with substitution - ########################################## + binary_cache = "file:///var/cache/binary-cache?secret-key=/etc/nix/secret-key" + # builtins.fetchurl is substituted missing_file = "/only-on-substituter.txt" substituter.succeed(f"echo 'this should only exist on the substituter' > {missing_file}") @@ -75,11 +70,8 @@ file_store_path = json.loads(file_store_path_json) - substituter.succeed(f"nix store sign --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {file_store_path}") - - importer.wait_for_unit("multi-user.target") + substituter.succeed(f"nix copy --to '{binary_cache}' {file_store_path}") - print("Testing fetchurl with substitution...") importer.succeed(f""" nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' builtins.fetchurl {{ @@ -88,26 +80,19 @@ }} ' """) - print("✓ fetchurl substitution works!") - - ########################################## - # Test 2: builtins.fetchTarball with substitution - ########################################## + # builtins.fetchTarball is substituted missing_tarball = "/only-on-substituter.tar.gz" - # Create a directory with some content substituter.succeed(""" mkdir -p /tmp/test-tarball echo 'Hello from tarball!' > /tmp/test-tarball/hello.txt echo 'Another file' > /tmp/test-tarball/file2.txt """) - - # Create a tarball substituter.succeed(f"tar czf {missing_tarball} -C /tmp test-tarball") - # For fetchTarball, we need to first fetch it without hash to get the store path, - # then compute the NAR hash of that path + # Fetch once without a hash to learn the store path, then derive the + # hashes the importer needs. tarball_store_path_json = substituter.succeed(f""" nix-instantiate --eval --json --read-write-mode --expr ' builtins.fetchTarball {{ @@ -118,22 +103,14 @@ tarball_store_path = json.loads(tarball_store_path_json) - # Get the NAR hash of the unpacked tarball in SRI format path_info_json = substituter.succeed(f"nix path-info --json-format 2 --json {tarball_store_path}").strip() path_info_dict = json.loads(path_info_json)["info"] - # narHash is already in SRI format tarball_hash_sri = path_info_dict[os.path.basename(tarball_store_path)]["narHash"] - print(f"Tarball NAR hash (SRI): {tarball_hash_sri}") - # Also get the old format hash for fetchTarball (which uses sha256 parameter) tarball_hash = substituter.succeed(f"nix-store --query --hash {tarball_store_path}").strip() - # Sign the tarball's store path - substituter.succeed(f"nix store sign --recursive --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {tarball_store_path}") + substituter.succeed(f"nix copy --to '{binary_cache}' {tarball_store_path}") - # Now try to fetch the same tarball on the importer - # The file doesn't exist locally, so it should be substituted - print("Testing fetchTarball with substitution...") result = importer.succeed(f""" nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' builtins.fetchTarball {{ @@ -144,23 +121,14 @@ """) result_path = json.loads(result) - print(f"✓ fetchTarball substitution works! Result: {result_path}") - # Verify the content is correct - # fetchTarball strips the top-level directory if there's only one content = importer.succeed(f"cat {result_path}/hello.txt").strip() assert content == "Hello from tarball!", f"Content mismatch: {content}" - print("✓ fetchTarball content verified!") - - ########################################## - # Test 3: Verify fetchTree does NOT substitute (preserves metadata) - ########################################## - - print("Testing that fetchTree without __final does NOT use substitution...") - # fetchTree with just narHash (not __final) should try to download, which will fail - # since the file doesn't exist on the importer - exit_code = importer.fail(f""" + # fetchTree does NOT substitute non-final inputs: without __final it + # must perform the real fetch (to preserve metadata like lastModified), + # so it fails since the file only exists on the substituter. + output = importer.fail(f""" nix-instantiate --eval --json --read-write-mode --expr ' builtins.fetchTree {{ type = "tarball"; @@ -170,9 +138,6 @@ ' 2>&1 """) - # Should fail with "does not exist" since it tries to download instead of substituting - assert "does not exist" in exit_code or "Couldn't open file" in exit_code, f"Expected download failure, got: {exit_code}" - print("✓ fetchTree correctly does NOT substitute non-final inputs!") - print(" (This preserves metadata like lastModified from the actual fetch)") + assert "does not exist" in output or "Couldn't open file" in output, f"Expected download failure, got: {output}" ''; } From 67e442bbbf0ea04b206305b290463ae210d515b4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 15:57:32 +0300 Subject: [PATCH 111/364] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz?narHash=sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E%3D' (2026-02-24) → 'https://releases.nixos.org/nixos/25.11/nixos-25.11.10470.0c88e1f2bdb9/nixexprs.tar.xz?narHash=sha256-amc4Y3GF3%2BanUi7IJeLVzf7hVqLb3ZqCGzYtkVyp7Qw%3D' (2026-05-05) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 1212049c3bea..95b771a30c1a 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1771903837, - "narHash": "sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E=", - "rev": "e764fc9a405871f1f6ca3d1394fb422e0a0c3951", + "lastModified": 1778003029, + "narHash": "sha256-amc4Y3GF3+anUi7IJeLVzf7hVqLb3ZqCGzYtkVyp7Qw=", + "rev": "0c88e1f2bdb93d5999019e99cb0e61e1fe2af4c5", "type": "tarball", - "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.10470.0c88e1f2bdb9/nixexprs.tar.xz" }, "original": { "type": "tarball", From 31f253b136d265bed12273f4a9de5388095936b4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:10:51 +0300 Subject: [PATCH 112/364] tests: Fix daemon compat tests for structured attrs --- tests/functional/structured-attrs.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 321cb6107992..d0b41670967b 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -51,15 +51,17 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " hacky=$(nix-instantiate --expr "$hackyExpr") nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}' -# Test warning for non-object exportReferencesGraph in structured attrs -# shellcheck disable=SC2016 -expectStderr 0 nix-build --no-out-link --expr ' - with import ./config.nix; - mkDerivation { - name = "export-graph-non-object"; - __structuredAttrs = true; - exportReferencesGraph = [ "foo" "bar" ]; - builder = "/bin/sh"; - args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; - } -' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" +if isDaemonNewer "2.34pre"; then + # Test warning for non-object exportReferencesGraph in structured attrs + # shellcheck disable=SC2016 + expectStderr 0 nix-build --no-out-link --expr ' + with import ./config.nix; + mkDerivation { + name = "export-graph-non-object"; + __structuredAttrs = true; + exportReferencesGraph = [ "foo" "bar" ]; + builder = "/bin/sh"; + args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; + } + ' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" +fi From acadae1ee65637bc6003813456b5cfe37c38fe05 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:19:42 +0300 Subject: [PATCH 113/364] packaging: Use libgit2 >= 1.9.3 It's been a while since libgit2 has been released. No notable features in this release though - just bugfixes. Unstable has been updated in [1]. [1]: https://github.com/NixOS/nixpkgs/pull/517329 --- packaging/dependencies.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b44f6ae46c80..68e44a925f79 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -62,6 +62,21 @@ scope: { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); }; + libgit2 = + if lib.versionAtLeast pkgs.libgit2.version "1.9.3" then + pkgs.libgit2 + else + # Grab newer libgit2. + pkgs.libgit2.overrideAttrs rec { + version = "1.9.3"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + tag = "v${version}"; + hash = "sha256-nJrRdPs86oGNL4W2CJb16oSUgfzYr9A2i5sw9BAehME="; + }; + }; + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { From bd2d65991100386983d9dbdd776e1afdb811383b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:26:22 +0300 Subject: [PATCH 114/364] packaging: Use mimalloc >= 3.3.2 --- packaging/dependencies.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 68e44a925f79..6a1873d9b3dd 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -17,16 +17,16 @@ scope: { inherit stdenv; mimalloc = - if lib.versionAtLeast pkgs.mimalloc.version "3.3.0" then + if lib.versionAtLeast pkgs.mimalloc.version "3.3.2" then pkgs.mimalloc else pkgs.mimalloc.overrideAttrs rec { - version = "3.3.0"; + version = "3.3.2"; src = pkgs.fetchFromGitHub { owner = "microsoft"; repo = "mimalloc"; tag = "v${version}"; - hash = "sha256-xy9gPihw3xvhnd6BrCYfMnnRp5dPSodynKRToYwxuzg="; + hash = "sha256-GZ37qQVDe9jgMb4Coe5oKvgaLTspZDlSkS5rdy1MfUU="; }; }; From c5aa96ef10ba7fffb581c5ea067a4f436ebe8283 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:29:15 +0300 Subject: [PATCH 115/364] packaging: Remove .broken override for libcurl Out nixpkgs is new enough now. --- packaging/dependencies.nix | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 6a1873d9b3dd..f531b709eec4 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -44,19 +44,13 @@ scope: { NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576"; }); - curl = - (pkgs.curl.override { - http3Support = !pkgs.stdenv.hostPlatform.isWindows; - # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. - zstdSupport = true; - brotliSupport = true; - zlibSupport = true; - }).overrideAttrs - { - # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. - # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. - meta.broken = false; - }; + curl = pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + }; libblake3 = pkgs.libblake3.override { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); From 3fd52d9a32ab70251dbad20575cc42b985c8cd5b Mon Sep 17 00:00:00 2001 From: Felix Stupp Date: Fri, 8 May 2026 22:05:58 +0000 Subject: [PATCH 116/364] doc: add documentation to __addErrorContext primop This adds .args and .doc fields to the primop registration for __addErrorContext, making it appear in the builtins reference manual. --- src/libexpr/primops.cc | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 297b2bf234de..08b4fd76acc5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1047,7 +1047,35 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value ** a static RegisterPrimOp primop_addErrorContext( PrimOp{ .name = "__addErrorContext", + .args = {"context", "value"}, .arity = 2, + .doc = R"( + Evaluate *context*, which can be coerced to a string, + and append it to any error or stack traces displayed while evaluating *value*. + Then return *value*. + + This function is useful for providing helpful context in complex Nix expressions + when the evaluation of *value* fails. + The additional context is applied when evaluating *value* itself fails, + not when attributes or elements of *value* are evaluated. + + For example, the module system from nixpkgs uses this to show + the relevant information about the options that were evaluating + when an error occurs. + + ```nix-repl + nix-repl> addErrorContext "while evaluating foo" (throw "bar") + error: + … while evaluating foo + + … while calling the 'throw' builtin + at «string»:1:56: + 1| with builtins; addErrorContext "while evaluating foo" (throw "bar") + | ^ + + error: bar + ``` + )", // The normal trace item is redundant .addTrace = false, .impl = prim_addErrorContext, From cabb895f190c33fda1871eb110547191f3481a06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 22:12:25 +0000 Subject: [PATCH 117/364] build(deps): bump cachix/install-nix-action from 31.10.5 to 31.10.6 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.5 to 31.10.6. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/ab739621df7a23f52766f9ccc97f38da6b7af14f...8aa03977d8d733052d78f4e008a241fd1dbf36b3) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49fff49b9737..2f294211ea74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: - name: Looking up the installer tarball URL id: installer-tarball-url run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -255,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From fcbe6cc5bffb374482b56d7b3445e2040a3972db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 22:12:30 +0000 Subject: [PATCH 118/364] build(deps): bump aws-actions/configure-aws-credentials Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 5.1.1 to 6.1.1. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/61815dcd50bd041e203e49132bacad1fd04d2708...d979d5b3a71173a29b74b5b88418bfda9437d885) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-version: 6.1.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/upload-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index f00dce4a5c6b..f7da2f66364d 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -34,7 +34,7 @@ jobs: # get the same uberhack that nix-shell has to support it. echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV" - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 with: role-to-assume: "arn:aws:iam::080433136561:role/nix-release" role-session-name: nix-release-oidc-${{ github.run_id }} From 5a870acfc9f5bccb9b50b2f6b1f541ecdfbfb1a6 Mon Sep 17 00:00:00 2001 From: znmz <267793835+znmz@users.noreply.github.com> Date: Mon, 11 May 2026 16:27:20 +0300 Subject: [PATCH 119/364] Fix typos in all .md, .sh, .nix files Also fixes one typo in each of the "remove_before_wrapper.py" "build-trace-entry-v3.yaml" "head.hbs" "sender.c" files --- doc/manual/remove_before_wrapper.py | 2 +- doc/manual/source/command-ref/env-common.md | 2 +- .../protocols/json/schema/build-trace-entry-v3.yaml | 2 +- doc/manual/source/release-notes/rl-2.30.md | 2 +- doc/manual/source/release-notes/rl-2.32.md | 2 +- doc/manual/source/store/file-system-object.md | 2 +- doc/manual/theme/head.hbs | 2 +- packaging/everything.nix | 2 +- scripts/create-darwin-volume.sh | 4 ++-- scripts/install-multi-user.sh | 4 ++-- tests/functional/binary-cache.sh | 2 +- tests/functional/build-delete.sh | 2 +- tests/functional/build.sh | 4 ++-- tests/functional/characterisation-test-infra.sh | 6 +++--- tests/functional/common/functions.sh | 2 +- tests/functional/dyn-drv/eval-outputOf.sh | 2 +- tests/functional/fetchTree-file.sh | 4 ++-- tests/functional/flakes/develop.sh | 10 +++++----- tests/functional/flakes/follow-paths.sh | 2 +- tests/functional/flakes/relative-paths.sh | 2 +- tests/functional/gc-closure.sh | 4 ++-- tests/functional/lang-gc.sh | 2 +- tests/functional/linux-sandbox.sh | 2 +- tests/functional/read-only-store.sh | 2 +- tests/functional/repl.sh | 2 +- tests/functional/simple.sh | 2 +- tests/functional/structured-attrs.sh | 2 +- tests/functional/suggestions.sh | 6 +++--- tests/nixos/ca-fd-leak/sender.c | 2 +- tests/nixos/fetch-git/test-cases/lfs/default.nix | 2 +- 30 files changed, 43 insertions(+), 43 deletions(-) diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py index 6da4c19b0ce7..a0fcb6a55776 100644 --- a/doc/manual/remove_before_wrapper.py +++ b/doc/manual/remove_before_wrapper.py @@ -22,7 +22,7 @@ def main(): shutil.rmtree(output, ignore_errors=True) shutil.rmtree(output_temp, ignore_errors=True) - # Execute nix command with `--write-to` tempary output + # Execute nix command with `--write-to` temporary output nix_command_write_to = nix_command + ['--write-to', output_temp] subprocess.run(nix_command_write_to, check=True) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 7ea1d0e5aa99..cc7fe77eae56 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -160,7 +160,7 @@ When [`use-xdg-base-directories`] is enabled, the configuration directory is res Likewise for the state and cache directories. -## Miscellanous environment variables +## Miscellaneous environment variables - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index c3a27d2a6e0b..3ff606672cf5 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -40,7 +40,7 @@ additionalProperties: false title: Build Trace Key description: | A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "key" part, refering to a derivation and output. + This is the "key" part, referring to a derivation and output. type: object required: - drvPath diff --git a/doc/manual/source/release-notes/rl-2.30.md b/doc/manual/source/release-notes/rl-2.30.md index 34d3e5bab4c6..5a65ed99af29 100644 --- a/doc/manual/source/release-notes/rl-2.30.md +++ b/doc/manual/source/release-notes/rl-2.30.md @@ -13,7 +13,7 @@ - Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220) The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`]. - However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. + However, by exploiting how structured attrs are implemented, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix. [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md index 5d90da0c9ebd..c59ecd6c2456 100644 --- a/doc/manual/source/release-notes/rl-2.32.md +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -8,7 +8,7 @@ - Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) - Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra parameter (the store directory). We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. diff --git a/doc/manual/source/store/file-system-object.md b/doc/manual/source/store/file-system-object.md index 60cb3e572063..2ffad5f0ec94 100644 --- a/doc/manual/source/store/file-system-object.md +++ b/doc/manual/source/store/file-system-object.md @@ -19,7 +19,7 @@ Every file system object is one of the following: In general, Nix does not assign any semantics to symbolic links. Certain operations however, may make additional assumptions and attempt to use the target to find another file system object. - > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. + > See [the Wikipedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. File system objects and their children form a tree. A bare file or symlink can be a root file system object. diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs index e514a99777f7..40bfef7d2f8f 100644 --- a/doc/manual/theme/head.hbs +++ b/doc/manual/theme/head.hbs @@ -11,5 +11,5 @@ MathJax = { } }; - + diff --git a/packaging/everything.nix b/packaging/everything.nix index df7d57a85860..74629d684036 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -105,7 +105,7 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = true; /** - `doCheck` controles whether tests are added as build gate for the combined package. + `doCheck` controls whether tests are added as build gate for the combined package. This includes both the unit tests and the functional tests, but not the integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). */ diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 7a61764d4f33..538a5e74d5ee 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -832,8 +832,8 @@ EOF # TODO: should probably alert the user if this is disabled? _sudo "to launch the Nix volume mounter" \ launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true - # TODO: confirm whether kickstart is necessesary? - # I feel a little superstitous, but it can guard + # TODO: confirm whether kickstart is necessary? + # I feel a little superstitious, but it can guard # against multiple problems (doesn't start, old # version still running for some reason...) _sudo "to launch the Nix volume mounter" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d4ea88b5ea6c..ae6625e1bc41 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -270,7 +270,7 @@ _diff() { printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" else - # simple colorized diff comatible w/ pre `--color` versions + # simple colorized diff compatible w/ pre `--color` versions diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" fi } @@ -961,7 +961,7 @@ configure_shell_profile() { cert_in_store() { # in a subshell # - change into the cert-file dir - # - get the phyiscal pwd + # - get the physical pwd # and test if this path is in the Nix store [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]] } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 68263459337e..d5b81523b30d 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -44,7 +44,7 @@ cacheDir2="$TEST_ROOT/binary+cache" nix copy --to "file://$cacheDir2" "$outPath" && [[ -d "$cacheDir2" ]] basicDownloadTests() { - # No uploading tests bcause upload with force HTTP doesn't work. + # No uploading tests because upload with force HTTP doesn't work. # By default, a binary cache doesn't support "nix-env -qas", but does # support installation. diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 66b14fd14384..e65615407aea 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -44,7 +44,7 @@ issue_6572_dependent_outputs() { # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - # Resolved derivations interferre with the deletion + # Resolved derivations interfere with the deletion nix-store --delete "${NIX_STORE_DIR}"/*.drv fi nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 0e76f949f55d..9f86f8100324 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -53,7 +53,7 @@ nix build -f multiple-outputs.nix --json nothing-to-install --no-link | jq --exi (.outputs | keys == ["out"])) ' -# But not when it's overriden. +# But not when it's overridden. nix build -f multiple-outputs.nix --json e^a_a --no-link nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' (.[0] | @@ -67,7 +67,7 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' (.outputs | keys == ["a_a", "b", "c"])) ' -# test buidling from non-drv attr path +# test building from non-drv attr path nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status ' (.[0] | diff --git a/tests/functional/characterisation-test-infra.sh b/tests/functional/characterisation-test-infra.sh index fecae29e8091..9651b4f7b6ba 100755 --- a/tests/functional/characterisation-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -19,14 +19,14 @@ cp "$TEST_ROOT/got" "$TEST_ROOT/expected" (( "$badDiff" == 0 )) ) -# matches empty, non-existant file is the same as empty file +# matches empty, non-existent file is the same as empty file echo -n > "$TEST_ROOT/got" ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" (( "$badDiff" == 0 )) ) -# doesn't matches non-empty, non-existant file is the same as empty file +# doesn't matches non-empty, non-existent file is the same as empty file echo Hi! > "$TEST_ROOT/got" ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" @@ -64,7 +64,7 @@ echo Bye! > "$TEST_ROOT/expected" (( "$badDiff" == 0 )) ) -# _NIX_TEST_ACCEPT matches empty, non-existant file not created +# _NIX_TEST_ACCEPT matches empty, non-existent file not created echo -n > "$TEST_ROOT/got" ( _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 771bbca785bc..0e571cc27a9a 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -101,7 +101,7 @@ killDaemon() { die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." fi - # Don't fail trying to stop a non-existant daemon twice. + # Don't fail trying to stop a non-existent daemon twice. if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh index 3681bd098899..13f7e7ba4c7e 100644 --- a/tests/functional/dyn-drv/eval-outputOf.sh +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -34,7 +34,7 @@ testStaticHello () { (assert a == b; null))' } -# Test with a regular old input-addresed derivation +# Test with a regular old input-addressed derivation # # `builtins.outputOf` works without ca-derivations and doesn't create a # placeholder but just returns the output path. diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh index 66be928c7b6b..a7c614b8ec0e 100755 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -76,12 +76,12 @@ EOF # For backwards compatibility, flake inputs that correspond to the # old 'tarball' fetcher should still have their type set to 'tarball' assert (nodes.tarball_default_unpack.locked.type == "tarball"); - # Unless explicitely specified, the 'unpack' parameter shouldn’t appear here + # Unless explicitly specified, the 'unpack' parameter shouldn’t appear here # because that would break older Nix versions assert (!nodes.tarball_default_unpack.locked ? unpack); assert (nodes.tarball_default_unpack.locked.narHash == "$input_directory_hash"); - # Explicitely passing the unpack parameter should enforce the desired behavior + # Explicitly passing the unpack parameter should enforce the desired behavior assert (nodes.no_ext_explicit_unpack.locked.narHash == nodes.tarball_default_unpack.locked.narHash); assert (nodes.tarball_explicit_no_unpack.locked.narHash == nodes.no_ext_default_no_unpack.locked.narHash); diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh index 0248adf49eaf..bb8850802742 100755 --- a/tests/functional/flakes/develop.sh +++ b/tests/functional/flakes/develop.sh @@ -66,7 +66,7 @@ echo "\$ENVVAR" EOF )" ]] -# Test wether `--keep-env-var` keeps the environment variable. +# Test whether `--keep-env-var` keeps the environment variable. ( expect='BAR' got="$(FOO='BAR' nix develop --ignore-env --keep-env-var FOO --no-write-lock-file .#hello <"$flakeFollowsA"/flake.nix < "$TEST_ROOT"/log grepQuietInverse 'error: renaming' "$TEST_ROOT"/log grepQuiet 'may not be deterministic' "$TEST_ROOT"/log diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index 8ccca2192af3..8bcf42cfc41d 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -4,7 +4,7 @@ source common.sh enableFeatures "read-only-local-store" -needLocalStore "cannot open store read-only when daemon has already opened it writeable" +needLocalStore "cannot open store read-only when daemon has already opened it writable" TODO_NixOS diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 9e752337f10d..cb181fbb6251 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -145,7 +145,7 @@ foo + baz ' "3" \ ./flake ./flake\#bar --experimental-features 'flakes' -# Test the `:reload` mechansim with flakes: +# Test the `:reload` mechanism with flakes: # - Eval `./flake#changingThing` # - Modify the flake # - Re-eval it diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index f6507d74182a..b7e8911930a4 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -20,7 +20,7 @@ text=$(cat "$outPath/hello") TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should -# be deleteable. +# be deletable. nix-store --delete "$outPath" [[ ! -e $outPath/hello ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index d0b41670967b..32d647a772eb 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -26,7 +26,7 @@ TODO_NixOS # following line fails. # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` -# (which is an associative array in thsi case) being fine. +# (which is an associative array in this case) being fine. # shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index fbca93da8590..e4324f2dfafc 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -30,7 +30,7 @@ EOF # Probable typo in the requested attribute path. Suggest some close possibilities NIX_BUILD_STDERR_WITH_SUGGESTIONS=$(! nix build .\#fob 2>&1 1>/dev/null) [[ "$NIX_BUILD_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ - fail "The nix build stderr should suggest the three closest possiblities" + fail "The nix build stderr should suggest the three closest possibilities" # None of the possible attributes is close to `bar`, so shouldn’t suggest anything NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) @@ -39,8 +39,8 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ - fail "The evaluator should suggest the three closest possiblities" + fail "The evaluator should suggest the three closest possibilities" NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '({ foo }: foo) { foo = 1; fob = 2; }' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean foo?" ]] || \ - fail "The evaluator should suggest the three closest possiblities" + fail "The evaluator should suggest the three closest possibilities" diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c index 639b88900228..f9fdd9405a52 100644 --- a/tests/nixos/ca-fd-leak/sender.c +++ b/tests/nixos/ca-fd-leak/sender.c @@ -62,6 +62,6 @@ int main(int argc, char ** argv) int buf; // Wait for the server to close the socket, implying that it has - // received the commmand. + // received the command. recv(sock, (void *) &buf, sizeof(int), 0); } diff --git a/tests/nixos/fetch-git/test-cases/lfs/default.nix b/tests/nixos/fetch-git/test-cases/lfs/default.nix index 289c3770911b..09b6fe44ffbd 100644 --- a/tests/nixos/fetch-git/test-cases/lfs/default.nix +++ b/tests/nixos/fetch-git/test-cases/lfs/default.nix @@ -172,7 +172,7 @@ f"did not set lfs, yet lfs-enrolled file is {file_size_default}b (>= 1KiB), probably bad default value" with subtest("Use as flake input"): - # May seem reduntant, but this has minor differences compared to raw + # May seem redundant, but this has minor differences compared to raw # fetchGit which caused failures before with TemporaryDirectory() as tempdir: client.succeed(f"mkdir -p {tempdir}") From 25295886ce8c4a8fa5ffd629540cac14a9736498 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 18:18:57 +0200 Subject: [PATCH 120/364] parseString(): Fix out-of-bounds read If the string isn't terminated, parseString() returns a string of size std::string::npos, which then causes an out-of-bounds read later. Fixes: ==47978== Invalid read of size 1 ==47978== at 0x4BEF70A: nix::expect(nix::(anonymous namespace)::StringViewStream&, char) (../src/libstore/derivations.cc:232) ==47978== by 0x4BEE3CA: parseDerivationOutput (../src/libstore/derivations.cc:383) ==47978== by 0x4BEE3CA: nix::parseDerivation(nix::StoreDirConfig const&, std::__cxx11::basic_string, std::allocator >&&, std::basic_string_view >, nix::ExperimentalFeatureSettings const&) (???:492) ==47978== by 0x3F3803: nix::DerivationTest_UnterminatedString_Test::TestBody() (../src/libstore-tests/derivation/external-formats.cc:27) ==47978== by 0x52AD3DD: void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*) (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x5298E3D: testing::Test::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x5298FCC: testing::TestInfo::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x529920E: testing::TestSuite::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x52A3996: testing::internal::UnitTestImpl::RunAllTests() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x52A3F74: testing::UnitTest::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x49DD52: RUN_ALL_TESTS (gtest.h:2334) ==47978== by 0x49DD52: main (???:16) --- src/libstore-tests/derivation/external-formats.cc | 8 ++++++++ src/libstore/derivations.cc | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 6fee675e9849..e31b3e85442f 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -15,6 +15,14 @@ TEST_F(DerivationTest, BadATerm_version) parseDerivation(*store, readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); } +TEST_F(DerivationTest, UnterminatedString) +{ + ASSERT_THROW( + parseDerivation( + *store, "Derive([(\"out\",\"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo", "bar", mockXpSettings), + FormatError); +} + TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 8475532bca33..37c89461de25 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -223,6 +223,7 @@ static BackedStringView parseString(StringViewStream & str) size_t start = 0; size_t end = str.remaining.size(); const auto data = str.remaining.data(); + bool foundClose = false; while (start < end) { auto idx = str.remaining.find('"', start); if (idx == std::string_view::npos) { @@ -233,10 +234,13 @@ static BackedStringView parseString(StringViewStream & str) ; if ((idx - pos) % 2 == 0) { // even number of backslashes end = idx; + foundClose = true; break; } start = idx + 1; } + if (!foundClose) + throw FormatError("unterminated string in derivation"); start = 0; const auto content = str.remaining.substr(start, end); From e6dab09c2f4b87a62dc7b0022548fa7939f688cd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 May 2026 12:01:04 -0400 Subject: [PATCH 121/364] NAR listing: always serialize `executable` field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously `NarListing::Regular::to_json` only included `"executable"` when true, while `MemorySourceAccessor::File::Regular::to_json` always included it. This inconsistency makes it harder for other implementations to use a single JSON definition for both — e.g. with Rust's `#[serde(derive)]` there's no way to have `skip_serializing_if` vary by generic parameter. Always serializing the field can only help clients (they get more information), never hurt them (any correct client already handles both `true` and `false`). --- src/libutil-tests/data/nar-listing/deep.json | 1 + src/libutil/memory-source-accessor/json.cc | 3 +- tests/functional/binary-cache.sh | 22 +++- tests/functional/nar-access.sh | 110 +++++++++++++++++-- 4 files changed, 126 insertions(+), 10 deletions(-) diff --git a/src/libutil-tests/data/nar-listing/deep.json b/src/libutil-tests/data/nar-listing/deep.json index a7ed47c4c030..f58974f66e40 100644 --- a/src/libutil-tests/data/nar-listing/deep.json +++ b/src/libutil-tests/data/nar-listing/deep.json @@ -15,6 +15,7 @@ "type": "directory" }, "foo": { + "executable": false, "size": 15, "type": "regular" } diff --git a/src/libutil/memory-source-accessor/json.cc b/src/libutil/memory-source-accessor/json.cc index 84fbb71bb2ef..ff3808d3c363 100644 --- a/src/libutil/memory-source-accessor/json.cc +++ b/src/libutil/memory-source-accessor/json.cc @@ -51,8 +51,7 @@ void adl_serializer::to_json(json & j, const NarListing::Re { if (r.contents.fileSize) j["size"] = *r.contents.fileSize; - if (r.executable) - j["executable"] = true; + j["executable"] = r.executable; if (r.contents.narOffset) j["narOffset"] = *r.contents.narOffset; } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 68263459337e..b76362bc2698 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -276,7 +276,27 @@ nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ - <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) + <(jq -S <<'EOF' +{ + "version": 1, + "root": { + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 4, + "narOffset": 232 + }, + "link": { + "type": "symlink", + "target": "xyzzy" + } + } + } +} +EOF + ) # Test debug info index generation. diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index cd419b4eefc8..f15a56d68867 100755 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -37,24 +37,120 @@ cp -r "$storePath" "$invalidPath" expect 1 nix store cat "$invalidPath/foo/baz" # Test --json. + +# Shallow listing of root (no --recursive) diff -u \ <(nix nar ls --json "$narFile" / | jq -S) \ - <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "foo": {}, + "foo-x": {}, + "qux": {}, + "zyx": {} + } +} +EOF + ) + +# Recursive listing of /foo from NAR (includes narOffset) diff -u \ <(nix nar ls --json -R "$narFile" /foo | jq -S) \ - <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 368 + }, + "baz": { + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 552 + }, + "data": { + "type": "regular", + "executable": false, + "size": 58, + "narOffset": 736 + } + } +} +EOF + ) + +# Single file from NAR diff -u \ <(nix nar ls --json -R "$narFile" /foo/bar | jq -S) \ - <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 368 +} +EOF + ) + +# Shallow listing from store diff -u \ <(nix store ls --json "$storePath" | jq -S) \ - <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "foo": {}, + "foo-x": {}, + "qux": {}, + "zyx": {} + } +} +EOF + ) + +# Recursive listing from store (no narOffset) diff -u \ <(nix store ls --json -R "$storePath/foo" | jq -S) \ - <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 0 + }, + "baz": { + "type": "regular", + "executable": false, + "size": 0 + }, + "data": { + "type": "regular", + "executable": false, + "size": 58 + } + } +} +EOF + ) + +# Single file from store diff -u \ - <(nix store ls --json -R "$storePath/foo/bar"| jq -S) \ - <(echo '{"type":"regular","size":0}' | jq -S) + <(nix store ls --json -R "$storePath/foo/bar" | jq -S) \ + <(jq -S <<'EOF' +{ + "type": "regular", + "executable": false, + "size": 0 +} +EOF + ) # Test missing files. expect 1 nix store ls --json -R "$storePath/xyzzy" 2>&1 | grep 'does not exist' From 8ecf74430e3e4035444cff133a5e86db355738b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 19:05:10 +0200 Subject: [PATCH 122/364] Fix posix_fallocate() error case On error, it returns errno directly, not -1. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0bc7b6e1b6c8..a963f2d3cdee 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -218,7 +218,7 @@ LocalStore::LocalStore(ref config) #if HAVE_POSIX_FALLOCATE res = posix_fallocate(fd.get(), 0, gcSettings.reservedSize); #endif - if (res == -1) { + if (res != 0) { writeFull(fd.get(), std::string(gcSettings.reservedSize, 'X')); [[gnu::unused]] auto res2 = From 033ac553ccd2959c053b292da8e89a991ef31cfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:10:28 +0200 Subject: [PATCH 123/364] Avoid size_t overflow --- src/libutil/serialise.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5345883ceb70..0477cd2c9a8d 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -28,14 +28,14 @@ void BufferedSink::operator()(std::string_view data) while (!data.empty()) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ - if (bufPos + data.size() >= bufSize) { + if (data.size() >= bufSize - bufPos) { flush(); writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ - size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size(); + size_t n = data.size() > bufSize - bufPos ? bufSize - bufPos : data.size(); memcpy(buffer.get() + bufPos, data.data(), n); data.remove_prefix(n); bufPos += n; From f61d9920d9206359ac12b0c080b2c8edca68cc7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:10:59 +0200 Subject: [PATCH 124/364] Avoid calling lseek() with an unintended negative offset --- src/libutil/serialise.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0477cd2c9a8d..d989f1d09055 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -287,6 +287,8 @@ void FdSource::skip(size_t len) #ifndef _WIN32 /* If we can, seek forward in the file to skip the rest. */ if (isSeekable && len) { + if (len > static_cast(std::numeric_limits::max())) + throw Error("cannot skip %d bytes: exceeds maximum file offset", len); if (lseek(fd, len, SEEK_CUR) == -1) { if (errno == ESPIPE) isSeekable = false; From 7afb83f0b694b9036f245a7cd8fad722bceca972 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:29:18 +0200 Subject: [PATCH 125/364] readError(): Replace assertions by exceptions We shouldn't crash on input from the other side. --- src/libutil/serialise.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d989f1d09055..7633732672c2 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -560,7 +560,8 @@ template StringSet readStrings(Source & source); Error readError(Source & source) { auto type = readString(source); - assert(type == "Error"); + if (type != "Error") + throw SerialisationError("unexpected error type '%s'", type); auto level = (Verbosity) readInt(source); [[maybe_unused]] auto name = readString(source); // removed auto msg = readString(source); @@ -569,11 +570,13 @@ Error readError(Source & source) .msg = HintFmt(msg), }; auto havePos = readNum(source); - assert(havePos == 0); + if (havePos != 0) + throw SerialisationError("deserializing error positions is not supported"); auto nrTraces = readNum(source); for (size_t i = 0; i < nrTraces; ++i) { havePos = readNum(source); - assert(havePos == 0); + if (havePos != 0) + throw SerialisationError("deserializing error positions is not supported"); info.traces.push_back(Trace{.hint = HintFmt(readString(source))}); } return Error(std::move(info)); From 22d1e6eef7eaaac25b322ac141bad07d25239357 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 30 Nov 2025 14:00:03 -0500 Subject: [PATCH 126/364] manual: More on stores, building and mounting in the file system Expand the manual on 1. Store paths: be more explicit about store path base names, introduce `{#digest}`, `{#name}`, and `{#base-name}` anchors, and add cross-links from `introduction.md`, `derivations.md`, `derivation-aterm.md`, `input-address.md`, and `rl-2.33.md`. 2. How store objects are exposed on the file system (new `os-file-system.md` page and new `{#exposing}` section of the store path page). 3. Building derivations: rewrite `building.md` with new sections on the build lifecycle, derivation resolution, file system setup (including `__structuredAttrs`/`passAsFile` file details), environment variables, and output processing. Try to be clearer about what *must* happen (spec, implementation agnostic), vs what Nix happens to do today. 4. Binary cache protocol: new `protocols/binary-cache/` section with an index page, a `.narinfo` format page (with field-by-field mapping to the JSON store object info schema), and move `nix-cache-info.md` into it. Link binary cache store types (HTTP, local, S3) back to the protocol page. 5. Store object metadata: new `{#metadata}` section in `store-object.md` linking to the JSON and narinfo formats. 6. Build trace: add `{#entry}` anchor, update JSON schema links. 7. Miscellaneous: fix typos in `os-file-system.md`, update `nix path-info --json-format` docs to cover v3, link `nix log` to store path base name and store object metadata, add `#store-directory` redirect, update stale links to match renamed anchors and moved pages. Co-authored-by: Eelco Dolstra Co-authored-by: Sergei Zimmerman --- doc/manual/redirects.json | 5 +- doc/manual/source/SUMMARY.md.in | 7 +- doc/manual/source/_redirects | 1 + doc/manual/source/command-ref/nix-hash.md | 20 +- .../source/command-ref/nix-prefetch-url.md | 2 +- doc/manual/source/glossary.md | 9 +- doc/manual/source/introduction.md | 2 +- .../source/language/advanced-attributes.md | 4 +- doc/manual/source/language/derivations.md | 2 +- .../binary-cache-substituter.md | 4 +- .../source/package-management/profiles.md | 3 +- .../source/protocols/binary-cache/index.md | 19 ++ .../source/protocols/binary-cache/narinfo.md | 42 +++ .../{ => binary-cache}/nix-cache-info.md | 4 +- .../source/protocols/derivation-aterm.md | 2 +- .../json/schema/build-trace-entry-v3.yaml | 6 +- .../json/schema/store-object-info-v3.yaml | 2 +- .../protocols/json/schema/store-path-v1.yaml | 2 +- doc/manual/source/protocols/nix32.md | 2 +- doc/manual/source/protocols/store-path.md | 6 +- doc/manual/source/release-notes/rl-2.33.md | 4 +- doc/manual/source/store/build-trace.md | 2 +- doc/manual/source/store/building.md | 246 +++++++++++++----- .../source/store/derivation/outputs/index.md | 2 +- .../store/derivation/outputs/input-address.md | 2 +- .../file-system-object/os-file-system.md | 40 +++ doc/manual/source/store/index.md | 31 ++- doc/manual/source/store/store-object.md | 6 + .../store/store-object/content-address.md | 2 +- doc/manual/source/store/store-path.md | 130 ++++++--- src/libstore/http-binary-cache-store.md | 2 +- .../include/nix/store/local-settings.hh | 2 +- src/libstore/local-binary-cache-store.md | 2 +- src/libstore/s3-binary-cache-store.md | 2 +- src/nix/flake.md | 4 +- src/nix/hash-convert.md | 4 +- src/nix/key-generate-secret.md | 2 +- src/nix/log.md | 14 +- src/nix/path-info.cc | 6 +- src/nix/path-info.md | 5 +- 40 files changed, 490 insertions(+), 162 deletions(-) create mode 100644 doc/manual/source/protocols/binary-cache/index.md create mode 100644 doc/manual/source/protocols/binary-cache/narinfo.md rename doc/manual/source/protocols/{ => binary-cache}/nix-cache-info.md (90%) create mode 100644 doc/manual/source/store/file-system-object/os-file-system.md diff --git a/doc/manual/redirects.json b/doc/manual/redirects.json index 0a6c71508006..329560ce786e 100644 --- a/doc/manual/redirects.json +++ b/doc/manual/redirects.json @@ -337,7 +337,7 @@ "string-literal": "string-literals.html" }, "language/derivations.html": { - "builder-execution": "../store/building.html#builder-execution" + "builder-execution": "../store/building.html" }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", @@ -363,6 +363,9 @@ "reverting": "contributing.html#reverting", "branches": "contributing.html#branches" }, + "store/store-path.html": { + "store-directory": "#store-directory-path" + }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", "package-attribute-set": "#package", diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b9020..3efe9a38de23 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -19,9 +19,10 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Exposing in OS File Systems](store/file-system-object/os-file-system.md) - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - - [Store Path](store/store-path.md) + - [Store Path and Store Directory](store/store-path.md) - [Store Derivation and Deriving Path](store/derivation/index.md) - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) @@ -137,7 +138,9 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - - [Nix Cache Info Format](protocols/nix-cache-info.md) + - [Binary Cache](protocols/binary-cache/index.md) + - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) + - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Nix32 Encoding](protocols/nix32.md) - [C API](c-api.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 7e4557f7d595..be82fe872d60 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -47,6 +47,7 @@ /package-management/package-management /package-management 301! /package-management/s3-substituter /store/types/s3-binary-cache-store 301! +/protocols/nix-cache-info /protocols/binary-cache/nix-cache-info 301! /protocols/protocols /protocols 301! /json/* /protocols/json/:splat 301! diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 7c17ce9095b4..c1a4251b0973 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -45,20 +45,20 @@ md5sum`. - `--base32` - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix + Print the hash in [Nix32](@docroot@/protocols/nix32.md) representation rather than hexadecimal. + This representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - `--base64` - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Similar to `--base32`, but print the hash in a [Base64](https://en.wikipedia.org/wiki/Base64) representation, + which is more compact than the Nix32 one. - `--sri` - Print the hash in SRI format with base-64 encoding. + Print the hash in [SRI](@docroot@/glossary.md#gloss-sri) format with Base64 encoding. The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. + followed by a hyphen (-) and the Base64 hash body. - `--truncate` @@ -71,18 +71,18 @@ md5sum`. - `--to-base16` - Don’t hash anything, but convert the base-32 hash representation + Don’t hash anything, but convert the [Nix32](@docroot@/protocols/nix32.md) hash representation *hash* to hexadecimal. - `--to-base32` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + *hash* to [Nix32](@docroot@/protocols/nix32.md). - `--to-base64` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + *hash* to Base64. - `--to-sri` @@ -134,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal, base-32, base-64, and SRI: +Converting between hexadecimal, Nix32, Base64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 diff --git a/doc/manual/source/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md index 8451778ad46d..86c20b9e1de4 100644 --- a/doc/manual/source/command-ref/nix-prefetch-url.md +++ b/doc/manual/source/command-ref/nix-prefetch-url.md @@ -32,7 +32,7 @@ Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. -The hash is printed using base-32 unless `--type md5` is specified, +The hash is printed using [Nix32](@docroot@/protocols/nix32.md) unless `--type md5` is specified, in which case it's printed using base-16. Additionally, if the option `--print-path` is used, the path of the downloaded file in the Nix store is also printed. diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index d7436a2052ee..112b0e3f8c0d 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -104,7 +104,7 @@ A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. - Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution). + Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md). This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. @@ -375,6 +375,13 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs +- [SRI]{#gloss-sri} + + [Subresource Integrity](https://www.w3.org/TR/SRI/) (SRI) is a [W3C specification](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) for integrity metadata. + Nix uses the SRI hash format (`-`) to specify content hashes in a way that is self-describing, since the hash algorithm is part of the format. + + [SRI]: #gloss-sri + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c917..33c8ba29faa9 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -49,7 +49,7 @@ builds correctly on your system, this is because you specified the dependency explicitly. This takes care of the build-time dependencies. Once a package is built, runtime dependencies are found by scanning -binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`). +binaries for the [hash parts](@docroot@/store/store-path.md#digest) of Nix store paths (such as `r8vvq9kq…`). This sounds risky, but it works extremely well. ## Multi-user support diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index 67612029c8a4..cc2743dc5ca9 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -337,8 +337,8 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](@docroot@/glossary.md#gloss-sri). + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of Base32 encoding. > **Note** > diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 2403183fc2d2..50aa525acbf4 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -165,7 +165,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > > for an Autoconf-style package. - The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + The name of an output is combined with the name of the derivation to create the [name part](@docroot@/store/store-path.md#name) of the output's store path, unless it is `out`, in which case just the name of the derivation is used. > **Example** > diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index e6a772213d6d..bc2cdfb27ab2 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,7 +19,7 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info @@ -28,7 +28,7 @@ WantMassQuery: 1 Priority: 30 ``` -When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/package-management/profiles.md b/doc/manual/source/package-management/profiles.md index 1d9e672a8def..53cf5061f834 100644 --- a/doc/manual/source/package-management/profiles.md +++ b/doc/manual/source/package-management/profiles.md @@ -11,8 +11,7 @@ in a directory another version might be stored in `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long strings prefixed to the directory names are cryptographic hashes (to be -precise, 160-bit truncations of SHA-256 hashes encoded in a base-32 -notation) of *all* inputs involved in building the package — sources, +precise, 160-bit truncations of SHA-256 hashes encoded in [Nix32](@docroot@/protocols/nix32.md)) of *all* inputs involved in building the package — sources, dependencies, compiler flags, and so on. So if two packages differ in any way, they end up in different locations in the file system, so they don’t interfere with each other. Here is what a part of a typical Nix diff --git a/doc/manual/source/protocols/binary-cache/index.md b/doc/manual/source/protocols/binary-cache/index.md new file mode 100644 index 000000000000..d86d307ad9ee --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/index.md @@ -0,0 +1,19 @@ +# Binary Cache + +The binary cache format is an interface designed for exposing a store over HTTP. + +A binary cache consists of: + +- A [`nix-cache-info`](./nix-cache-info.md) file at the root with remote-side configuration. +- For each [store object](@docroot@/store/store-object.md): + - A [`.narinfo`](./narinfo.md) file containing the object's [metadata](@docroot@/store/store-object.md#metadata) and a (usually relative) URL to the corresponding compressed NAR. + - A possibly-compressed [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) containing the store object's file system data. +- For every entry in the [build trace](@docroot@/store/build-trace.md), a JSON file at `build-trace-v2//.doi`: + - the path encodes the [key](@docroot@/protocols/json/build-trace-entry.md#key) + - the contents are the [value](@docroot@/protocols/json/build-trace-entry.md#value). + +The following [store types](@docroot@/store/types/index.md) use the binary cache format: + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) — served over HTTP(S) +- [Local Binary Cache Store](@docroot@/store/types/local-binary-cache-store.md) — stored on the file system +- [S3 Binary Cache Store](@docroot@/store/types/s3-binary-cache-store.md) — stored in an AWS S3 bucket diff --git a/doc/manual/source/protocols/binary-cache/narinfo.md b/doc/manual/source/protocols/binary-cache/narinfo.md new file mode 100644 index 000000000000..e2e2efac0eeb --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/narinfo.md @@ -0,0 +1,42 @@ +# `.narinfo` Format + +A `.narinfo` file contains the [metadata of a store object](@docroot@/store/store-object.md#metadata) in the [binary cache](@docroot@/protocols/binary-cache/index.md) format. +It is a simple line-oriented format where each line is a `Key: Value` pair. +Some keys (e.g. `Sig`) may appear multiple times. + +The file is named `.narinfo`, where `` is the [hash part](@docroot@/store/store-path.md#digest) of the store object's [store path](@docroot@/store/store-path.md). + +The fields correspond to those documented in the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format: + +| `.narinfo` field | JSON field | Differences | +|---|---|---| +| `StorePath` | [`path`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_path) | Full [store path](@docroot@/store/store-path.md) rather than [store path base name](@docroot@/store/store-path.md#base-name) | +| `URL` | [`url`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_url) | | +| `Compression` | [`compression`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_compression) | Defaults to `bzip2` if omitted | +| `FileHash` | [`downloadHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadHash) | String-encoded hash rather than structured | +| `FileSize` | [`downloadSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadSize) | | +| `NarHash` | [`narHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narHash) | String-encoded hash rather than structured | +| `NarSize` | [`narSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narSize) | | +| `References` | [`references`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_references) | Space-separated [store path base names](@docroot@/store/store-path.md#base-name) rather than a JSON array | +| `Deriver` | [`deriver`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_deriver) | [Store path base name](@docroot@/store/store-path.md#base-name); `unknown-deriver` instead of `null` | +| `Sig` | [`signatures`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_signatures) | May appear multiple times rather than using an array | +| `CA` | [`ca`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_ca) | String-encoded [content address](@docroot@/store/store-object/content-address.md) rather than structured | + +## Example + + + +``` +StorePath: /nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +URL: nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz?sha256=1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3 +Compression: xz +FileHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +FileSize: 4029176 +NarHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +NarSize: 34878 +References: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +Deriver: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv +Sig: asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +Sig: qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +CA: fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh +``` diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/binary-cache/nix-cache-info.md similarity index 90% rename from doc/manual/source/protocols/nix-cache-info.md rename to doc/manual/source/protocols/binary-cache/nix-cache-info.md index e8351e1cebe8..3859b2a10ac4 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/binary-cache/nix-cache-info.md @@ -1,6 +1,6 @@ -# Nix Cache Info Format +# `nix-cache-info` Format -The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/protocols/binary-cache/index.md) (e.g., `https://cache.example.com/nix-cache-info`). MIME type: `text/x-nix-cache-info` diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 523678e663e3..778614eb1602 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -26,7 +26,7 @@ Derivations are serialised in one of the following formats: When derivation is encoded to a [store object] we make the following choices: -- The store path name is the derivation name with `.drv` suffixed at the end +- The store path [name](@docroot@/store/store-path.md#name) is the derivation name with `.drv` suffixed at the end Indeed, the ATerm format above does *not* contain the name of the derivation, on the assumption that a store path will also be provided out-of-band. diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index 3ff606672cf5..2e825af6fb8a 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -4,7 +4,7 @@ title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). + This schema describes the JSON representation of an [entry](@docroot@/store/build-trace.md#entry) in a [build trace](@docroot@/store/build-trace.md). > **Warning** > @@ -39,7 +39,7 @@ additionalProperties: false key: title: Build Trace Key description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "key" part, referring to a derivation and output. type: object required: @@ -61,7 +61,7 @@ additionalProperties: false value: title: Build Trace Value description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "value" part, describing an output. type: object required: diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index e0be716ef781..e0c1ffef134f 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -109,7 +109,7 @@ $defs: type: string title: Store Directory description: | - The [store directory](@docroot@/store/store-path.md#store-directory) this store object belongs to (e.g. `/nix/store`). + The [path to the store directory](@docroot@/store/store-path.md#store-directory-path) this store object belongs within (e.g. `/nix/store`). additionalProperties: false impure: diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index f1f58c2bf1ac..b1251c7426e9 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -24,7 +24,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md) (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md index 72afe893ea24..d8da1e9952cf 100644 --- a/doc/manual/source/protocols/nix32.md +++ b/doc/manual/source/protocols/nix32.md @@ -1,6 +1,6 @@ # Nix32 Encoding -Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. +Nix32 is Nix's variant of [Base32](https://en.wikipedia.org/wiki/Base32) encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. ## Alphabet diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 1aa79615d1c8..bdce19d1d62c 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -18,11 +18,9 @@ where - `name` = the name of the store object. -- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) +- `store-dir` = the [path of the store directory](@docroot@/store/store-path.md#store-directory-path) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. - - Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). +- `digest` = [Nix32](@docroot@/protocols/nix32.md) representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed697029389..668230693b8d 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -149,9 +149,9 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js } ``` - The map from store path base names to store object info is nested under the `info` field. + The map from [store path base names](@docroot@/store/store-path.md#base-name) to store object info is nested under the `info` field. -- **Store path base names instead of full paths**: +- **[Store path base names](@docroot@/store/store-path.md#base-name) instead of full paths**: Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths. Combined with `storeDir`, the full path can be reconstructed. diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index a879d37d208d..cb9cb3099680 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -8,7 +8,7 @@ The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. It maps the inputs of builds to the outputs of builds. -Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. +Each *[entry]{#entry}* in the build trace maps a [derivation][derivation] to a map of [output] names to [store objects][store object]. In general the derivations used as a key should be [*resolved*](./resolution.md). A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 32e800129342..087413406487 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -1,72 +1,178 @@ # Building -## Normalizing derivation inputs +As discussed in the [main page on derivations](./derivation/index.md): -- Each input must be [realised] prior to building the derivation in question. +> A derivation is a specification for running an executable on precisely defined input to produce one or more [store objects][store object]. + +This page describes *building* a derivation, which is to say following the instructions in the derivation to actually run the executable. +Some elements of derivations are self-explanatory. +For example, the arguments specified in the derivation really are the arguments passed to the executable. +In other cases, however, there is additional common steps performed by Nix for all derivations --- mostly for setting up the build environment and collecting the built outputs. + +The chief design consideration for the building process is *determinism*. +Conventional operating systems are typically not designed with determinism in mind. +But determinism is needed to make Nix's build caching a transparent abstraction. + +> **Explanation** +> +> For example, no one wants to slightly modify a derivation, and then find that it no longer builds for an unrelated reason, because the original derivation *also* doesn't build anymore, but the cache hit on the original derivation was hiding this. +> We want builds that succeed once to continue succeeding, to encourage fearless modification of old build recipes. +> Determinism is what enables things that once worked to keep working. + +The life cycle of a build can be broken down into 3 parts: + +1. Spawn the builder process with the proper environment, including the correct process arguments, environment variables, and file system state. + +2. Wait for the builder process to exit and collect its exit status. + Exit code 0 means success; anything else is a build failure. + (Strictly speaking, Nix detects process exit by waiting for the standard output and error streams to close. + If a builder explicitly closes these streams without exiting, Nix will kill it, and deem the build a failure. + Processes should therefore exit *without* explicitly closing those standard streams, and let the exiting of the process close them implicitly.) + + Nix also logs the standard output and error of the process, but this is just for human convenience and does not influence the behavior of the system. + (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. + In practice, Nix will store the log in `/nix/var/log/nix`) + +3. Processing the outputs after the builder has exited. + + The builder process on exit should have left behind files for each output the derivation is supposed to produce. + The files must be processed to turn them into bona fide store objects. + If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Step (3) is done by Nix externally to the build itself, which is just steps (1) and (2). +In step (3), just inert data is processed, since the builder process has exited or been killed by then. +Step (1) however is best described not from Nix's perspective, but from the build process's perspective. + +> **Explanation** +> +> Ultimately, what matters for determinism is what the build process can observe: what resources (files, networking, etc.) it can see, what syscalls succeed or fail, etc. +> Nix can achieve this through many different sandboxing strategies (namespaces, VMs, chroots, ...), but the process shouldn't be able to tell them apart. +> We therefore specify building from the process's perspective, not Nix's perspective, to focus on *what*, not *how*. + +## What derivations can be built + +Actually only some derivations are ready to be built. +In particular, only [*resolved*](./resolution.md) derivations can be built. +That is to say, a derivation that depends on other derivations is not ready yet to be built, because some of those other derivations might not have yet been built. +If the other derivations are indeed all built, we can witness this fact by resolving the derivation, and converting all the derivation's input references into plain store paths. + +> **Note** +> +> Note that [input-addressing](derivation/outputs/input-address.md) derivations are improperly resolved. +> As discussed on the linked page, the current input-addressing algorithm does not respect resolution-equivalence of derivations (\\(\\sim_\mathrm{Drv}\\)). +> That means that if Nix properly resolved an input-addressed derivation, the resolved derivation would have different input addresses, violating expectations. +> Nix therefore improperly resolves the derivation, keeping its original input-addressed output paths, creating an invalid derivation that is both resolved and instructed to create the outputs at the originally expected paths. + +## Environment of the builder process + +This section describes how the [`builder`](./derivation/index.md#builder) is executed. + +> **Implementation detail** +> +> Nix prevents multiple [Nix instances][Nix instance] from performing the same build at the same time, for example by acquiring exclusive file locks. + +### File system + +The builder should have access to a limited file system where only certain objects are available. +The most important exposed files are the inputs (other store objects) of the (resolved) derivation. +Additionally, some other files are exposed. + +#### Store inputs + +The builder will be run against a file system in which the [store directory][store directory path] contains the [closure] of the inputs. +In particular, consider a store that just contains this closure. +That store is exposed to the file system according to the rules specified in the [Exposing Store Objects in OS File Systems](./store-path.md#exposing) documentation. +This precisely defines the file system layout of the store that should be visible to the builder process. + +> **Note** +> +> Historically, Nix exposed *at least* the following store contents to the builder, but also arbitrarily other store objects, due to limitations around operating systems' file system virtualization capabilities, and wanting to avoid copying or moving files. +> It still can do this in so-called *unsandboxed* builds. +> +> Such builds should be considered discouraged, but one that works less badly against non-mischievous derivations than might be expected. +> This is because store paths are relatively unpredictable, so a well-behaved program is unlikely to stumble upon a store object it wasn't supposed to know about. +> +> As operating systems developed better file system primitives, the need for disabling sandboxing has lessened greatly over the years, and this trend should continue into the future. + +The outputs are expected to be created in that store directory as if they were valid store objects. +(They are just files during builder execution, but during [processing outputs](#processing-outputs) they will be turned into proper store objects.) +The [environment variables](#env-vars) for each output indicate where the builder should write them; +Nix ensures that those paths do not yet exist when the builder is run. + +> **Note** +> +> In sandboxed builds, ensuring that the outputs do not exist in the store directory is trivial. +> In unsandboxed builds, it is harder in general. +> In the worst case, the derivation is in fact rewritten so different output paths are used instead, and then the outputs are rewritten back to the intended output paths after. +> In the content-addressing case rewriting would be needed either way, but in the input-addressing case, this is a significant degradation, as the point of input addressing is to avoid rewrites by knowing output paths in advance. [realised]: @docroot@/glossary.md#gloss-realise +[closure]: @docroot@/glossary.md#gloss-closure +[store directory path]: ./store-path.md#store-directory-path + +### Other file system state + +- The current working directory of the builder process will be a fresh temporary directory. + It is initially empty when the process starts except for a few input files: -- Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. + - If [`__structuredAttrs`](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs) is enabled: `.attrs.json` (the derivation attributes as JSON) and `.attrs.sh` (a Bash-compatible rendering of the same). + The environment variables `NIX_ATTRS_JSON_FILE` and `NIX_ATTRS_SH_FILE` point to these files, respectively. -## Builder Execution {#builder-execution} + - If [`passAsFile`](@docroot@/language/advanced-attributes.md#adv-attr-passAsFile) is used (only without `__structuredAttrs`): for each attribute name listed, a file `.attr-` where `` is the [Nix32](@docroot@/protocols/nix32.md)-encoded SHA-256 hash of the attribute name. + The environment variable `Path` points to the file containing the attribute's value. -The [`builder`](./derivation/index.md#builder) is executed as follows: + In sandboxed builds, this directory is at a deterministic path inside the sandbox (controlled by the [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir) setting, default `/build`). + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for the host-side location. -- A temporary directory is created where the build will take place. The - current directory is changed to this directory. +- Basic device nodes for essential operations (null device, random number generation, standard streams as a pseudo terminal) - See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information. + (A pseudo terminal would not be strictly necessary since the standard streams are passively logging, not there to facilitate interaction. + But it is still useful to entice programs to do nicer logging with e.g. colors etc.) -- The environment is cleared and set to the derivation attributes, as - specified above. +- On Linux: Process information via `/proc` -- In addition, the following variables are set: +- Minimal user and group identity information - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. +- A loopback-only network configuration with hostname set to `localhost` - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. +> **Note** +> +> Fixed-output derivations have access to additional operating system state to facilitate communication with the outside world, such as network name resolution and TLS certificate verification. +> This is necessary because these derivations are allowed to access the network, unlike regular derivations which are fully sandboxed. - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. +### Environment variables {#env-vars} - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. +The environment is cleared and set to the derivation attributes, as +specified above. - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). +For most derivations types this must contain at least: - - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the derivation. A detailed explanation of this - behavior can be found in the - [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). +- For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. + Each output path is a concatenation of the cryptographic hash of all build inputs, the `name` attribute and the output name. + (The output name is omitted if it's `out`.) - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +In addition, the following variables are set: -- If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple [Nix instances][Nix instance] from performing the same - build at the same time. +- `NIX_BUILD_TOP` contains the path of the temporary directory for this build. -- A log of the combined standard output and error is written to - `/nix/var/log/nix`. +- Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the temporary directory. + This is to prevent the builder from accidentally writing temporary files anywhere else. + Doing so might cause interference by other processes. -- The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. +- `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. -- The temporary directory is removed (unless the `-K` option was - specified). +- `HOME` is set to `/homeless-shelter`. + (Without sandboxing, this discourages programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity.) + Usually, when `HOME` is set, it is used as the location of the home directory, even if it points to a non-existent path. + +- `NIX_STORE` is set to the path of the top-level Nix [store directory path] (typically, `/nix/store`). + +- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` is set to `true` for the derivation. + A detailed explanation of this behavior can be found in the [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). + +### Arguments + +The builder is passed the arguments specified by the derivation attribute `args`. ## Processing outputs @@ -74,28 +180,40 @@ If the builder exited successfully, the following steps happen in order to turn - **Normalize the file permissions** - Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Any possible `setuid` and `setgid` - bits are cleared. - - > **Note** - > - > Setuid and setgid programs are not currently supported by Nix. - > This is because the Nix archives used in deployment have no concept of ownership information, - > and because it makes the build result dependent on the user performing the build. + The files must conform to the model described in the [Exposing in OS file systems](./file-system-object/os-file-system.md) section. + For example, timestamps and permissions are canonicalised. - **Calculate the references** - Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Nix scans each output path for [references] to input store objects by looking for the [digest][store path digest] of each input. + (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) + Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. + + Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. + + The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). + (This is not a special restriction for outputs; it is true for the references of all store objects in general.) + + In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. + For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. + Scratch store paths must therefore be used instead. + Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. - Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. - If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. +At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. +Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. +> **Implementation detail** +> +> Nix will normally clean up and remove the temporary build directory after every build, successful or unsuccessful. +> The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. +> The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. +[references]: ./store-object.md#references +[store path digest]: ./store-path.md#digest +[store object]: ./store-object.md [Nix instance]: @docroot@/glossary.md#gloss-nix-instance +[content-addressing]: ./derivation/outputs/content-address.md +[input-addressing]: ./derivation/outputs/input-address.md +[fixed content-addressing]: ./derivation/outputs/content-address.md#fixed +[floating content-addressing]: ./derivation/outputs/content-address.md#floating +[build trace]: ./build-trace.md diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index ca2ce6665b04..9b46405cdf58 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -9,7 +9,7 @@ The outputs specification is a map, from names to specifications for individual ## Output Names {#outputs} -Output names can be any string which is also a valid [store path](@docroot@/store/store-path.md) name. +Output names can be any string which is also a valid [store path name](@docroot@/store/store-path.md#name). The name mapped to each output specification is not actually the name of the output. In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. However, an output spec named "out" describes and output store object whose name is just the derivation name. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 3fd20f17d724..6df9b94961e8 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -16,7 +16,7 @@ Concretely, this would cause a "mass rebuild" whenever any fetching detail chang To solve this problem, we compute output hashes differently, so that certain output hashes become identical. We call this concept quotient hashing, in reference to quotient types or sets. -So how do we compute the hash part of the output paths of an input-addressed derivation? +So how do we compute the [hash part](@docroot@/store/store-path.md#digest) of the output paths of an input-addressed derivation? This is done by the function `hashQuotientDerivation`, shown below. First, a word on inputs. diff --git a/doc/manual/source/store/file-system-object/os-file-system.md b/doc/manual/source/store/file-system-object/os-file-system.md new file mode 100644 index 000000000000..6ce21f7788c9 --- /dev/null +++ b/doc/manual/source/store/file-system-object/os-file-system.md @@ -0,0 +1,40 @@ +# Exposing File System Objects in real operating system file systems + +Nix's [file system object] data model is minimal. +All the various other bits and pieces of real world filesystem interfaces, such as [extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes), are specifically ignored to reduce our interface surface and the reproducibility issues associated with a larger interface. +In the view of Nix's developers, the types of simple, fine-grained batch jobs (typically, building software) that Nix specializes in simply don't benefit enough from that extra complexity for it to be worth the costs of supporting it. + +But to actually be used by software, file system objects need to be made available through the operating system's file system. +This is sometimes called "mounting" or "exposing" the file system object, though do note it may or may not be implemented with what the operating system calls "mounting". + +[file system object]: ../file-system-object.md + +## Metadata normalization + +File systems typically contain other metadata that is outside Nix's data model. +To avoid this other metadata being a side channel and source of nondeterminism, Nix is careful to normalize to fixed values. +For example, on Unix, the following metadata normalization occurs: + +- The creation and last modification timestamps on all files are set to Unix Epoch 1s (00:00:01 1/1/1970 UTC) + +- The group is set to the [default group](@docroot@/command-ref/conf-file.md#conf-build-users-group) + +- The Unix mode of the file to 0444 or 0555 (i.e., read-only, with execute permission enabled if the file was originally executable). + +- Any possible `setuid` and `setgid` bits are cleared. + + > **Note** + > + > `setuid` and `setgid` programs are not currently supported by Nix. + > These special file system permissions are in general a security footgun, and with data owned by different users in different stores, it would especially be a hazard when copying store objects between stores. + > + > This restriction has not proved to be onerous in practice. + > For example, NixOS uses so called setuid-wrappers which are outside the store. + +> **Explanation** +> +> As discussed before, Nix essentially shares its file system object data model with other tools like Git. +> But those tools tend to ignore this metadata in both directions --- when reading files, like Nix, but when writing files, timestamps are set organically, and the user is free to set other special permissions (`setuid`, `setgid`, sticky, etc.) however they like. +> Normalizing, and not just ignoring, this metadata is therefore what distinguishes Nix from these other tools more than the file system object data model itself. +> +> Nix's approach is motivated by deterministic building. Whereas Git can assume that humans running commands will simply ignore timestamps etc. as appropriate, understanding they are local and ephemeral, Nix aims to run software that was not necessarily designed with Nix in mind, and is unaware of whatever sandboxing/virtualization is in place. diff --git a/doc/manual/source/store/index.md b/doc/manual/source/store/index.md index f1e8f1402988..d063fc4fdc05 100644 --- a/doc/manual/source/store/index.md +++ b/doc/manual/source/store/index.md @@ -2,4 +2,33 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). +Concretely, albeit using concepts that are only defined in the rest of the chapter, a store consists of: + +- A set of [store objects][store object], the immutable file system data. + + This can also be looked at as a map from [store paths][store path] to store objects. + +- A set of [derivations][derivation], instructions for building store objects. + + This can also be looked at as a map from [store paths][store path] to derivations. + Since store paths to derivations always end in `.drv`, and store paths to other store objects never do, the two maps can also be combined into one. + Derivations can also be encoded as store objects too. + +- A [build trace], a record of which derivations have been built and what they produced. + + > **Warning** + > + > The concept of a build trace is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +There are [multiple types of Nix stores][store type] with different capabilities, such as the default one on the [local file system][local store] (`/nix/store`) or [binary caches][binary cache]. + +[store object]: ./store-object.md +[store path]: ./store-path.md +[derivation]: ./derivation/index.md +[build trace]: ./build-trace.md + +[store type]: ./types/index.md +[local store]: ./types/local-store.md +[binary cache]: ./types/http-binary-cache-store.md diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 170d6246cfe7..eb84ab84370d 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -66,3 +66,9 @@ A store can only contain a store object if it also contains all the store object > > The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. > However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. + +### Store Object Metadata {#metadata} + +[Store implementations](@docroot@/store/types/index.md) currently associate more information than described above with a store object. +Quite arguably some of this information doesn't belong here, because it conflates concerns. +For details see the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format or the [narinfo](@docroot@/protocols/binary-cache/narinfo.md) format. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 94d94ec6d4ae..282b7545a231 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -9,7 +9,7 @@ In particular, the content-addressing scheme will ensure that the digest of the - file system object graph (the root one and its children, if it has any) - references -- [store directory](../store-path.md#store-directory) +- [store directory path](../store-path.md#store-directory-path) - name of the store object, and not any other information, which would not be an intrinsic property of that store object. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 04bdfec004c2..43037f7baac6 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -1,72 +1,136 @@ -# Store Path +# Store Path and Store Directory -> **Example** -> -> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` -> -> A rendered store path +Nix's [store object] and [file system object] data models are minimal and abstract. +But to actually be used by software, store objects need to be made available through the operating system's file system. + +This is done by exposing all the store objects in a single *[store directory][store directory path]*. +Every entry in that directory is a *[store path base name]* pointing to a store object. +Store objects exposed in this way can then be referenced by *[store paths][store path]*. + +[store object]: ./store-object.md +[file system object]: ./file-system-object.md +[store path]: #store-path +[store path base name]: #base-name +[store directory path]: #store-directory-path + +## Store Path Base Name {#base-name} -Nix implements references to [store objects](./store-object.md) as *store paths*. +Nix implements references to store objects as *store path base names*. -Think of a store path as an [opaque], [unique identifier]: -The only way to obtain store path is by adding or building store objects. -A store path will always reference exactly one store object. +Think of a store path base name as an [opaque], [unique identifier]: +The only way to obtain a store path base name is by adding or building store objects. +A store path base name will always reference exactly one store object. [opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -Store paths are pairs of +Store path base names are pairs of -- A 20-byte digest for identification -- A symbolic name for people to read +- A 20-byte [digest]{#digest} for identification +- A symbolic [name]{#name} for people to read > **Example** > > - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq` > - Name: `firefox-33.1` -To make store objects accessible to operating system processes, stores have to expose store objects through the file system. +A store path base name is rendered to a string as the concatenation of -A store path is rendered to a file system path as the concatenation of - -- [Store directory](#store-directory) (typically `/nix/store`) -- Path separator (`/`) -- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) +- Digest rendered in [Nix32], a variant of [Base32] (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name > **Example** > > ``` -> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 -> |--------| |------------------------------| |----------| -> store directory digest name +> q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |------------------------------| |----------| +> digest name > ``` -Exactly how the digest is calculated depends on the type of store path. +[Nix32]: @docroot@/protocols/nix32.md +[Base32]: https://en.wikipedia.org/wiki/Base32 + +Exactly how the digest is calculated depends on the type of store object being referenced. Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). -## Store Directory - -Every [Nix store](./index.md) has a store directory. +## Store Directory Path -Not every store can be accessed through the file system. -But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). +Every [Nix store] has a store directory path. +This is an absolute, lexically canonical (not containing any `..`, `.`, or similar) path which points to the directory where all store objects are to be found. -[file system objects]: ./file-system-object.md - -This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +[Nix store]: ./index.md > **Note** > > The store directory defaults to `/nix/store`, but is in principle arbitrary. -It is important which store a given store object belongs to: +## Store Path + +A store path is the pair of a store directory path and a [store path base name]. +It is rendered to a file system path as the concatenation of + +- [Store directory path] (typically `/nix/store`) +- Path separator (`/`) +- The [store path base name] + +> **Example** +> +> ``` +> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +When we have fixed a given store, or given store directory path (that all the stores in use share), the abstract syntax for store paths and the abstract syntax for store path base names coincide: the store directory path is known from context, so only the other two fields vary from one store path to the next. + +## Exposing Store Objects in OS File Systems {#exposing} + +Not every store can be accessed through the file system. +But if the store has a file system representation, the following should be true: + +- The store directory path is canonical: no prefix of the path (i.e. path of the first *n* path segments) points to a symlink. + In other words, the store directory can be looked up from the store directory path without following any symlinks. + (This condition is a separate condition in addition to the "lexical canonicity" described above, which is a property of just the path itself. + This (regular) "canonicity" is a property about the path and the filesystem it navigates jointly.) + + > **Note** + > + > The [`allow-symlinked-store`](@docroot@/command-ref/conf-file.md#conf-allow-symlinked-store) setting can be used to relax this requirement. + +- The store directory path in fact points to a directory. + +- The store directory contains, for every store object in the store, the [file system object] of that store object at the (rendered) [store path base name]. + The permissions and other metadata for these files in the store directory is in the normal form described in [Exposing in OS file systems](./file-system-object/os-file-system.md). + +The above properties mean that the following file accesses will work. +Suppose we have a store available on the file system per the above rules, and `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` is the store path base name of a store object in that store. + +- Suppose that the store directory (path) is `/foo/bar`. + Then, `/foo/bar/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` exists and is the file system object of that store object. + +- Suppose that we don't know what the store directory path of the store is, but we do have a capability `storeDir` to the store directory on the file system. + (This would be a "file descriptor" on Unix, or a "file handle" on Windows.) + Then (using the Unix notation for this): + ``` + openat(storeDir, "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1", O_NOFOLLOW) + ``` + will succeed (so long as the file system object is not a symlink), and the yielded capability will point to the file system object of that store object. + + (The behavior for symlinks is harder to specify because of limitations in POSIX.) + +## Relocating store objects + +The inclusion of the store directory path in the full rendered store path means that the full rendered store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +(And actually, all of the currently-supported ways of computing the digest of a store path also depend on the store directory path, as described in the [specification of store path digests](@docroot@/protocols/store-path.md). +So this is also true even just for store path base names, in general.) + +It is therefore important to consider which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee referential integrity if store paths do not cross store boundaries. -Therefore one can only copy store objects to a different store if +One can only copy store objects to a different store if - The source and target stores' directories match diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md index 20c26d0c2caf..03dd350ec518 100644 --- a/src/libstore/http-binary-cache-store.md +++ b/src/libstore/http-binary-cache-store.md @@ -2,7 +2,7 @@ R"( **Store URL format**: `http://...`, `https://...` -This store allows a binary cache to be accessed via the HTTP +This store allows a [binary cache](@docroot@/protocols/binary-cache/index.md) to be accessed via the HTTP protocol. )" diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 7381b5b8e766..dc4322753d7f 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -196,7 +196,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#env-vars) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism. - From 87c3c3251c4a1ac39bac6772342bca00cceb18f7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 21:48:22 +0300 Subject: [PATCH 264/364] Fix truncated store hash part in the manual introduction, document FreeBSD support --- doc/manual/source/introduction.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c917..51f9fb45d2e0 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -10,7 +10,7 @@ as /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/ -where `b6gvzjyb2pg0…` is a unique identifier for the package that +where `q06x3jll2yfz…` is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features. @@ -174,7 +174,7 @@ the package: ## Portability -Nix runs on Linux and macOS. +Nix runs on Linux, macOS and FreeBSD. ## NixOS From 3d445e7bc597cd78d28a3c0f74fc9bc5b7e8ba03 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 21:48:52 +0300 Subject: [PATCH 265/364] Bring prerequisites-source.md up to date This has bitrotten a lot and needed a lot of updates since the meson migration. I've removed version bounds that we don't check in the build system, which have had likely bitrotten too. Also start to follow SEMBR in these docs. --- .../installation/prerequisites-source.md | 126 +++++++----------- 1 file changed, 49 insertions(+), 77 deletions(-) diff --git a/doc/manual/source/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md index 640032c5176b..e98067348ee6 100644 --- a/doc/manual/source/installation/prerequisites-source.md +++ b/doc/manual/source/installation/prerequisites-source.md @@ -1,80 +1,52 @@ # Prerequisites - - GNU Autoconf () and the - autoconf-archive macro collection - (). These are - needed to run the bootstrap script. - - - GNU Make. - - - Bash Shell. The `./configure` script relies on bashisms, so Bash is - required. - - - A version of GCC or Clang that supports C++23. - - - `pkg-config` to locate dependencies. If your distribution does not - provide it, you can get it from - . - - - The OpenSSL library to calculate cryptographic hashes. If your - distribution does not provide it, you can get it from - . - - - The `libbrotlienc` and `libbrotlidec` libraries to provide - implementation of the Brotli compression algorithm. They are - available for download from the official repository - . - - - cURL and its library. If your distribution does not provide it, you - can get it from . - - - The SQLite embedded database library, version 3.6.19 or higher. If - your distribution does not provide it, please install it from - . - - - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). - - To enable it, install - `pkgconfig` and the Boehm garbage collector, and pass the flag - `--enable-gc` to `configure`. - - - The `boost` library of version 1.66.0 or higher. It can be obtained - from the official web site . - - - The `editline` library of version 1.14.0 or higher. It can be - obtained from the its repository - . - - - The `libsodium` library for verifying cryptographic signatures - of contents fetched from binary caches. - It can be obtained from the official web site - . - - - Recent versions of Bison and Flex to build the parser. (This is - because Nix needs GLR support in Bison and reentrancy support in - Flex.) For Bison, you need version 2.6, which can be obtained from - the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex, - you need version 2.5.35, which is available on - [SourceForge](http://lex.sourceforge.net/). Slightly older versions - may also work, but ancient versions like the ubiquitous 2.5.4a - won't. - - - The `libseccomp` is used to provide syscall filtering on Linux. This - is an optional dependency and can be disabled passing a - `--disable-seccomp-sandboxing` option to the `configure` script (Not - recommended unless your system doesn't support `libseccomp`). To get - the library, visit . - - - On 64-bit x86 machines only, `libcpuid` library - is used to determine which microarchitecture levels are supported + This list and lower version bounds are maintained on best-effort basis. When in doubt, check the `meson.build` files. + + - Meson build system (). + + - Ninja (). + + - A version of GCC or Clang that supports C++23 (anything newer than Clang 19 or GCC 14 is likely to work). + + - `pkg-config` to locate dependencies. + If your distribution does not provide it, you can get it from . + + - The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + - The `libbrotlienc` and `libbrotlidec` libraries to provide implementation of the Brotli compression algorithm. + They are available for download from the official repository . + + - cURL library. + If your distribution does not provide it, you can get it from . + + - The SQLite embedded database library, version 3.6.19 or higher. + If your distribution does not provide it, please install it from . + + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce the evaluator’s memory consumption (optional). + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the option `-Dlibexpr:gc=enabled` to `meson setup`. + + - The `boost` library of version 1.87.0 or higher. + It can be obtained from the official web site . + + - The `editline` library of version 1.14.0 or higher. + It can be obtained from the its repository . + + - The `libsodium` library for verifying cryptographic signatures of contents fetched from binary caches. + It can be obtained from the official web site . + + - Recent versions of Bison and Flex to build the parser. + (This is because Nix needs C++ template support in Bison and reentrancy support in Flex.) + + - The `libseccomp` is used to provide syscall filtering on Linux. + This is an optional dependency and can be disabled passing a `-Dlibstore:seccomp-sandboxing=disabled` option to the `meson setup` command + (Not recommended unless your system doesn't support `libseccomp`). + To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library is used to determine which microarchitecture levels are supported (e.g., as whether to have `x86_64-v2-linux` among additional system types). - The library is available from its homepage - . - This is an optional dependency and can be disabled - by providing a `--disable-cpuid` to the `configure` script. - - - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and - RapidCheck are required, which are available at - and - respectively. + The library is available from its homepage . + This is an optional dependency and can be disabled by providing a `-Dlibutil:cpuid=disabled` option to `meson setup` script. + + - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at + and respectively. From 5d2600a01dfc0e722ddbb66d9ae3b4b65cec41ae Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Wed, 17 Jun 2026 13:49:11 -0400 Subject: [PATCH 266/364] Fix exceptions not being caught in nix::Pid::~Pid Using function try blocks with destructors causes the exception to be rethrown instead of properly caught. --- src/libutil/unix/processes.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 7accf2e1759e..32d59535883a 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -50,11 +50,13 @@ Pid::Pid(pid_t pid) } Pid::~Pid() -try { - if (pid != -1) - kill(/*allowInterrupts=*/false); -} catch (...) { - ignoreExceptionInDestructor(); +{ + try { + if (pid != -1) + kill(/*allowInterrupts=*/false); + } catch (...) { + ignoreExceptionInDestructor(); + } } void Pid::operator=(pid_t pid) From c37e17c280934a6ca667984151516b0bb6212de4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 20:35:42 +0300 Subject: [PATCH 267/364] Increase the sourceToSink/sinkToSource coroutine stack sizes We had too little headroom previously, since the default stack size if 128KiB and half of that is typically consumed by a 64KiB buffer for copying between Source/Sink. 512KiB should be more than enough for the limit that we have (64 levels in NARs , which usually also bounds the recursion depth with some constant factor). --- src/libutil/archive.cc | 3 +-- src/libutil/serialise.cc | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 8d9b833c29aa..56172bf32b9a 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -36,8 +36,7 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); /* Maximum directory nesting depth for dumpPath()/parseDump(). Bounds stack usage so deep trees cannot overflow the (possibly coroutine) - stack these run on. Chosen to fit comfortably in the default 128 KiB - boost coroutine stack. */ + stack these run on. */ static constexpr size_t narMaxDepth = 64; PathFilter defaultPathFilter = [](const std::string &) { return true; }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0529c2ca2f89..eb1209843179 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -369,6 +369,13 @@ void StringSource::skip(size_t len) pos += len; } +/* 512KiB is a conservative estimate for deeply nested NARs, which are limited + to 64 levels. We also tend to allocate rather large buffers on the stack, so + we should leave plenty of headroom. Note that no evaluation is supposed to + happen on sourceToSink/sinkToSource coroutine stacks (for Boehm GC reasons), + which requires much more stack space. */ +static constexpr size_t defaultCoroutineStackSize = 512 * 1024; + std::unique_ptr sourceToSink(fun reader) { struct SourceToSink : FinishSink @@ -392,8 +399,9 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = - coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { + coro = coro_t::push_type( + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), + [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -450,8 +458,9 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = - coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { + coro = coro_t::pull_type( + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), + [&](coro_t::push_type & yield) { /* Feed the consumer in chunks, instead of on each write to avoid excessive context switching. parseDump does lots of small writes to the sink, which we should From 9586b80af2c75d9ffefc01402a4e15c63847ebbd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 17 Jun 2026 21:04:34 +0300 Subject: [PATCH 268/364] Print attributes in repl commands (:env or :st) in lexicographic order Depending on how nix is built, the order of variables in the environments can differ [1], due to static initialisation order. Technically, in non-unity builds the order is fully undefined due to the static initialisation order fiasco. I recently added tests for this functionality and they started failing in nixpkgs, because it doesn't use unity builds. [1]: https://github.com/NixOS/nixpkgs/pull/532575 --- src/libexpr/eval.cc | 27 ++++++++++++------- .../repl/debugger-fail-throw-env-1.expected | 2 +- .../repl/debugger-fail-throw-env-2.expected | 2 +- .../repl/debugger-okay-ignore-try.expected | 8 +++--- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 81ed77c90593..069d7e2c5190 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -667,27 +667,34 @@ std::optional EvalState::getDoc(Value & v) return {}; } +static StaticEnv::Vars lexicographicOrder(const SymbolTable & st, StaticEnv::Vars vars) +{ + std::ranges::sort(vars, [&st](const auto & lhs, const auto & rhs) { + return std::string_view(st[lhs.first]) < std::string_view(st[rhs.first]); + }); + return vars; +} + // just for the current level of StaticEnv, not the whole chain. -void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) +static void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { std::cout << ANSI_MAGENTA; - for (auto & i : se.vars) - std::cout << st[i.first] << " "; + for (auto & [name, displacement] : lexicographicOrder(st, se.vars)) + std::cout << st[name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } // just for the current level of Env, not the whole chain. -void printWithBindings(const SymbolTable & st, const Env & env) +static void printWithBindings(const SymbolTable & st, const Env & env) { if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - auto j = env.values[0]->attrs()->begin(); - while (j != env.values[0]->attrs()->end()) { - std::cout << st[j->name] << " "; - ++j; - } + auto * bindings = env.values[0]->attrs(); + /* TODO: Don't print the whole attribute set, since it can be quite large. */ + for (const Attr * attr : bindings->lexicographicOrder(st)) + std::cout << st[attr->name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } @@ -708,7 +715,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << ANSI_MAGENTA; // for the top level, don't print the double underscore ones; // they are in builtins. - for (auto & i : se.vars) + for (auto & i : lexicographicOrder(st, se.vars)) if (!hasPrefix(st[i.first], "__")) std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; diff --git a/tests/functional/repl/debugger-fail-throw-env-1.expected b/tests/functional/repl/debugger-fail-throw-env-1.expected index 865aa58eaf12..921395b8f928 100644 --- a/tests/functional/repl/debugger-fail-throw-env-1.expected +++ b/tests/functional/repl/debugger-fail-throw-env-1.expected @@ -8,7 +8,7 @@ Env level 0 static: _ Env level 1 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :quit error: diff --git a/tests/functional/repl/debugger-fail-throw-env-2.expected b/tests/functional/repl/debugger-fail-throw-env-2.expected index 251da649f3ec..3eb47bdf53aa 100644 --- a/tests/functional/repl/debugger-fail-throw-env-2.expected +++ b/tests/functional/repl/debugger-fail-throw-env-2.expected @@ -15,7 +15,7 @@ Env level 2 static: x Env level 3 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :bt diff --git a/tests/functional/repl/debugger-okay-ignore-try.expected b/tests/functional/repl/debugger-okay-ignore-try.expected index 23a5f476ac41..7d4e72fdaeab 100644 --- a/tests/functional/repl/debugger-okay-ignore-try.expected +++ b/tests/functional/repl/debugger-okay-ignore-try.expected @@ -29,7 +29,7 @@ Env level 0 static: someFailingExpr tried Env level 1 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: undefined variable 'bricked' @@ -54,7 +54,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered @@ -85,7 +85,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered @@ -110,7 +110,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered From 1ba3bc047c6d0494f13c4138268bca8392c1ad4c Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Wed, 17 Jun 2026 20:12:52 +0100 Subject: [PATCH 269/364] worker-settings: speed factor is non-integer The value of the speed factor when specifying a remote builder can be any positive number. Correct the comments / documentation to reflect that fact. --- src/libstore/include/nix/store/worker-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/worker-settings.hh b/src/libstore/include/nix/store/worker-settings.hh index 32c2b3684269..d78ca420f776 100644 --- a/src/libstore/include/nix/store/worker-settings.hh +++ b/src/libstore/include/nix/store/worker-settings.hh @@ -185,7 +185,7 @@ public: 4. The maximum number of builds that Nix executes in parallel on the machine. Typically this should be equal to the number of CPU cores. - 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer or decimal number. If there are multiple machines of the right type, Nix prefers the fastest, taking load into account. 6. A comma-separated list of supported [system features](#conf-system-features). From f88c306340939391bdec8e3cd9d753c4f6f5ccf2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 23:09:28 +0300 Subject: [PATCH 270/364] Unbreak the formatting of the builtins in the manual 2a71fbf41017b4b9ad87f7c6fa5506b80743e1a7 broke the manual. Also the documentation was quite strange in some places (and slightly inaccurate). Using inline documentation is much more readable and doesn't have the formatting footguns. --- src/libexpr/primops.cc | 199 ++++++----------------------------------- 1 file changed, 28 insertions(+), 171 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 951c6e74607b..1a837a1b96ff 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3090,11 +3090,7 @@ static RegisterPrimOp primop_attrNames({ alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrNames, }); @@ -3128,11 +3124,7 @@ static RegisterPrimOp primop_attrValues({ Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrValues, }); @@ -3159,9 +3151,7 @@ static RegisterPrimOp primop_getAttr({ the `.` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_getAttr, }); @@ -3251,9 +3241,7 @@ static RegisterPrimOp primop_hasAttr({ `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_hasAttr, }); @@ -3314,12 +3302,7 @@ static RegisterPrimOp primop_removeAttrs({ evaluates to `{ y = 2; }`. - # Time Complexity - - O(n + k log k) where: - - n = number of attributes in input set - k = number of attribute names to remove + Has `O(n + k log k)` time complexity, where `n` is number of attributes in the *set* and `k` is the size of *list*. )", .impl = prim_removeAttrs, }); @@ -3408,9 +3391,7 @@ static RegisterPrimOp primop_listToAttrs({ { foo = 123; bar = 456; } ``` - # Time Complexity - - O(n log n) where n = number of list elements + Has `O(n log n)` time complexity, where `n` is size of the list. )", .impl = prim_listToAttrs, }); @@ -3487,12 +3468,7 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - # Time Complexity - - O(n * log m) where: - - n = number of attributes in the smaller set - m = number of attributes in the larger set + Has `O(n log m)` time complexity, where `n` and `m` are the sizes of the smallest and largest set respectively. )", .impl = prim_intersectAttrs, }); @@ -3533,12 +3509,7 @@ static RegisterPrimOp primop_catAttrs({ evaluates to `[1 2]`. - # Time Complexity - - O(n * log m) where: - - n = list length - m = number of attributes per set + Has `O(n)` time complexity, where `n` is the size of the *list*. )", .impl = prim_catAttrs, }); @@ -3583,9 +3554,7 @@ static RegisterPrimOp primop_functionArgs({ the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. - # Time Complexity - - O(n) where n = number of formal arguments + Has constant time complexity. )", .impl = prim_functionArgs, }); @@ -3619,13 +3588,9 @@ static RegisterPrimOp primop_mapAttrs({ evaluates to `{ a = 10; b = 20; }`. - # Time Complexity - - O(n) where: - - n = number of attributes - - Calls to `f` are performed afterwards, when needed. + Has `O(n)` time complexity, where `n` is the size of the *attrset*. + Note that no calls to *f* are performed by the builtin. + The function *f* is called on demand when a resulting attribute value is evaluated. )", .impl = prim_mapAttrs, }); @@ -3714,12 +3679,7 @@ static RegisterPrimOp primop_zipAttrsWith({ } ``` - # Time Complexity - - O(N * log k) where: - - N = total attributes across all sets - k = number of unique keys across all sets + Has `O(n log n)` time complexity, where `n` is the number of attributes across all sets. )", .impl = prim_zipAttrsWith, }); @@ -3787,9 +3747,7 @@ static RegisterPrimOp primop_head({ isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. - # Time Complexity - - O(1) + Has constant time complexity. )", .impl = prim_head, }); @@ -3821,10 +3779,6 @@ static RegisterPrimOp primop_tail({ > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. - - # Time Complexity - - O(n) where n = list length (copies n-1 elements) )", .impl = prim_tail, }); @@ -3860,13 +3814,9 @@ static RegisterPrimOp primop_map({ evaluates to `[ "foobar" "foobla" "fooabc" ]`. - # Time Complexity - - O(n) where: - - n = list length - - Calls to `f` are performed afterwards when needed. + Has `O(n)` time complexity, where `n` is the size of the *list*. + Note that no calls to *f* are performed by the builtin, but *f* itself is evaluated and its type is checked eagerly. + The function *f* is called on demand when a resulting list element is evaluated. )", .impl = prim_map, }); @@ -3916,13 +3866,7 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. - - # Time Complexity - - O(n * T_f) (eager; predicate is forced) where: - - n = list length - T_f = predicate evaluation time + Has linear time complexity in the size of the input *list*. )", .impl = prim_filter, }); @@ -3946,15 +3890,7 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. - - # Time Complexity - - O(n * T) (worst case) where: - - n = list length - T = time to compare two elements - - returns early if the elements is found + Short-circuits and does not evaluate elements that occur in the list after the first match. )", .impl = prim_elem, }); @@ -3972,12 +3908,6 @@ static RegisterPrimOp primop_concatLists({ .args = {"lists"}, .doc = R"( Concatenate a list of lists into a single list. - - # Time Complexity - - O(N) where: - - N = total number of elements across all lists )", .impl = prim_concatLists, }); @@ -3994,10 +3924,6 @@ static RegisterPrimOp primop_length({ .args = {"e"}, .doc = R"( Return the length of the list *e*. - - # Time Complexity - - O(1) )", .impl = prim_length, }); @@ -4063,12 +3989,7 @@ static RegisterPrimOp primop_foldlStrict({ but lacks these benefits. See also [Nixpkgs `lib.foldl`](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.lists.foldl). - # Time Complexity - - O(n * T_op) where: - - n = list length - T_op = `op` call evaluation time + Has linear time complexity in the size of the list. )", .impl = prim_foldlStrict, }); @@ -4107,15 +4028,7 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_pred) where: - - - n = `list` length - - T_pred = `pred` call evaluation time - - returns early when `pred` returns `true` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `true`. )", .impl = prim_any, }); @@ -4131,15 +4044,7 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_f) where: - - - n = list length - - T_f = predicate evaluation time - - returns early when `pred` returns `false` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `false`. )", .impl = prim_all, }); @@ -4179,16 +4084,7 @@ static RegisterPrimOp primop_genList({ returns the list `[ 0 1 4 9 16 ]`. - # Time Complexity - - Complexity of `genList generator n`: O(n) - - Complexity of `deepSeq (genList generator n)`: O(n * T_f) - - where: - - n = requested length - T_f = `generator` call evaluation time + Has linear time complexity. )", .impl = prim_genList, }); @@ -4300,15 +4196,8 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. - # Time Complexity - - O(n log n * T_cmp), where: - - n = `list` length - T_cmp = `comparator` call evaluation time - - Uses an adaptive sort that exploits existing sorted runs in the input, - down to O(n * T_cmp) when the list is already sorted. + Runs in `O(n log n)` time on average, where `n` is the size of the *list*. + Uses an adaptive sort that exploits existing sorted runs in the input, down to `O(n)` when the list is already sorted. )", .impl = prim_sort, }); @@ -4371,12 +4260,7 @@ static RegisterPrimOp primop_partition({ { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` - # Time Complexity - - O(n * T_pred) where: - - n = list length - T_pred = `pred` call evaluation time + Runs in linear time in the size of the *list*. )", .impl = prim_partition, }); @@ -4431,13 +4315,7 @@ static RegisterPrimOp primop_groupBy({ { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` - # Time Complexity - - O(N * T_f + N * log k) where: - - N = number of `list` elements - T_f = `f` call evaluation time - k = number of unique groups + Has `O(n log n)` time complexity, where `n` is the size of the input *list*. )", .impl = prim_groupBy, }); @@ -4480,14 +4358,6 @@ static RegisterPrimOp primop_concatMap({ .doc = R"( This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. - - # Time Complexity - - O(k * T_f + N) where: - - k = length of input list - T_f = time to call `f` on an element - N = total number of elements returned by `f` calls )", .impl = prim_concatMap, }); @@ -5191,13 +5061,6 @@ static RegisterPrimOp primop_concatStringsSep({ Concatenate a list of strings with a separator between each element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. - - # Time Complexity - - O(n + m) (amortized) where: - - n = number of list elements - m = total length of output string )", .impl = prim_concatStringsSep, }); @@ -5283,13 +5146,7 @@ static RegisterPrimOp primop_replaceStrings({ evaluates to `"fabir"`. - # Time Complexity - - O(n * k * c) (worst case) where: - - n = length of input string - k = number of replacement patterns - c = average length of patterns in 'from' list + Has `O(n k)` time complexity, where `n` is the length of *s* and `k` is the number of replacements. )", .impl = prim_replaceStrings, }); From b308d8cea8f79fb2d0cee2c5732524f3c68ba4ea Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Jun 2026 22:50:23 +0200 Subject: [PATCH 271/364] doc: Generalize the JSON guidelines to "Data Modeling Guidelines" They were kind of the same thing and we've been applying them more broadly on occasion. --- doc/manual/source/SUMMARY.md.in | 2 +- doc/manual/source/_redirects | 3 ++- .../{json-guideline.md => data-modeling.md} | 11 ++++++++--- doc/manual/source/release-notes/rl-2.23.md | 4 ++-- doc/manual/source/release-notes/rl-2.33.md | 2 +- src/libmain/include/nix/main/common-args.hh | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) rename doc/manual/source/development/{json-guideline.md => data-modeling.md} (86%) diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b9020..a73446a99021 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -149,7 +149,7 @@ - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - - [JSON guideline](development/json-guideline.md) + - [Data Modeling Guidelines](development/data-modeling.md) - [C++ style guide](development/cxx.md) - [Static Analysis](development/static-analysis.md) - [Experimental Features](development/experimental-features.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 7e4557f7d595..17aba776a376 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -27,7 +27,8 @@ /contributing/documentation /development/documentation 301! /contributing/experimental-features /development/experimental-features 301! /contributing/cli-guideline /development/cli-guideline 301! -/contributing/json-guideline /development/json-guideline 301! +/contributing/json-guideline /development/data-modeling 301! +/development/json-guideline /development/data-modeling 301! /contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! diff --git a/doc/manual/source/development/json-guideline.md b/doc/manual/source/development/data-modeling.md similarity index 86% rename from doc/manual/source/development/json-guideline.md rename to doc/manual/source/development/data-modeling.md index 309b4b3a06e4..a9d131147588 100644 --- a/doc/manual/source/development/json-guideline.md +++ b/doc/manual/source/development/data-modeling.md @@ -1,7 +1,12 @@ -# JSON guideline +# Data Modeling Guidelines -Nix consumes and produces JSON in a variety of contexts. -These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. +Nix consumes and produces JSON and attribute sets in a variety of contexts. +These guidelines ensure consistent practices for our interfaces, for ease of use, and so that experience in one part carries over to another. + +For these guidelines, we will use JSON terminology, but they apply equally well to new attribute set interfaces (primops, etc.). +Note that these are guidelines first and foremost. Exceptions include: +- Feature testing: e.g., it is OK to do `builtins?frobnicate`. +- Compatibility: we generally do not change stable interfaces just to make them comply. New replacements can be added with care. ## Extensibility diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index b358a0fdc3c3..92e5f4599440 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). + The JSON format for derivations has been slightly revised to better conform to our [data modeling guidelines](@docroot@/development/data-modeling.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + the [data modeling guidelines](@docroot@/development/data-modeling.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed697029389..cc5781f37314 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -135,7 +135,7 @@ This is the legacy format, preserved for backwards compatibility: ### Version 2 (`--json-format 2`) -The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes: +The new structured format follows the [data modeling guidelines](@docroot@/development/data-modeling.md) with the following changes: - **Nested structure with top-level metadata**: diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c47..b20df3a99ec9 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -81,8 +81,8 @@ struct MixPrintJSON : virtual Args * This is a template to avoid accidental coercions from `string` to `json` in the caller, * to avoid mistakenly passing an already serialized JSON to this function. * - * It is not recommended to print a JSON string - see the JSON guidelines - * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * It is not recommended to print a JSON string - see the data modeling guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/data-modeling.html - * but you _can_ print a sole JSON string by explicitly coercing it to * `nlohmann::json` first. */ From 5a3d3986e6af6a6b43027ef0c767f3d39d855505 Mon Sep 17 00:00:00 2001 From: Tom Hunze Date: Sat, 4 Apr 2026 12:18:03 +0200 Subject: [PATCH 272/364] nix develop: use `runPhase` to run phases, fall back to `runHook` Using `runHook`, `nix develop --phase ` still executes the generic `Phase` when `Phase` is defined when calling `stdenv.mkDerivation` [1]. `runPhase` was specifically added to avoid this problem and to be more consistent with `genericBuild` behavior [2]. [1] https://github.com/NixOS/nix/issues/6202 [2] https://github.com/NixOS/nixpkgs/pull/230874 --- src/nix/develop.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 50b248bcb884..3105efa82536 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -605,7 +605,13 @@ struct CmdDevelop : Common, MixEnvironment // FIXME: foundMakefile is set by buildPhase, need to get // rid of that. script += fmt("foundMakefile=1\n"); - script += fmt("runHook %1%Phase\n", *phase); + script += + fmt("if declare -f runPhase >/dev/null; then\n" + " runPhase %1%Phase\n" + "else\n" + " runHook %1%Phase\n" + "fi\n", + *phase); } else if (!command.empty()) { From 2662dc95aff7b9f77ddf00421d9354de2c57d284 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 18 Jun 2026 23:52:40 +0300 Subject: [PATCH 273/364] Remove UnkeyedValidPathInfo::id This doesn't seem to be used anywhere now. It got added initially in 762cee72ccd860e72c7b639a1dd542ac0f298bb2, where it was used in registerValidPaths. But nowadays we do queryValidPathId for that use-case unconditionally. That can be improved to avoid some unnecessary SQLite queries in case we know the primary id, but it can be done in a much more local way that's not exposed in the interface. --- src/libstore/include/nix/store/path-info.hh | 8 -------- src/libstore/local-store.cc | 4 +--- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 7ba30359b547..0dd19b81ecf1 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -90,14 +90,6 @@ struct UnkeyedValidPathInfo */ uint64_t narSize = 0; - /** - * internal use only: SQL primary key for on-disk store objects with - * `LocalStore`. - * - * @todo Remove, layer violation - */ - uint64_t id = 0; - /** * Whether the path is ultimately trusted, that is, it's a * derivation output that was built locally. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4a4ffc988a8..dd5d325239d2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -814,8 +814,6 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto info = std::make_shared(path, UnkeyedValidPathInfo(*this, narHash)); - info->id = id; - info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); @@ -836,7 +834,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s info->ca = ContentAddress::parseOpt(s); /* Get the references. */ - auto useQueryReferences(state.stmts->QueryReferences.use().apply(info->id)); + auto useQueryReferences(state.stmts->QueryReferences.use().apply(id)); while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); From acfc7d845176c7c6a514ff8527be7266850d495c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Jun 2026 01:41:36 +0300 Subject: [PATCH 274/364] Add test for broken positions on repl :load-file This is my SNAFU from a091a8100a8. I neglected the fact that :l and a bit of other things also nuke the cache. The following commit will have the fix. --- tests/functional/repl/file-b.nix | 9 ++++++- .../functional/repl/load-and-reload.expected | 21 ++++++++++++++++ ...on-existent-file.in => load-and-reload.in} | 6 +++-- .../reload-with-non-existent-file.expected | 24 ------------------- 4 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 tests/functional/repl/load-and-reload.expected rename tests/functional/repl/{reload-with-non-existent-file.in => load-and-reload.in} (70%) delete mode 100644 tests/functional/repl/reload-with-non-existent-file.expected diff --git a/tests/functional/repl/file-b.nix b/tests/functional/repl/file-b.nix index ddde63c16c42..7416216c9f51 100644 --- a/tests/functional/repl/file-b.nix +++ b/tests/functional/repl/file-b.nix @@ -1 +1,8 @@ -{ fromB = 2; } +{ + fromBFails = throw "b"; + fromB = 2; + /** + Some documentation. + */ + funcFromB = x: x; +} diff --git a/tests/functional/repl/load-and-reload.expected b/tests/functional/repl/load-and-reload.expected new file mode 100644 index 000000000000..1642987fe6bc --- /dev/null +++ b/tests/functional/repl/load-and-reload.expected @@ -0,0 +1,21 @@ +Nix +Type :? for help. + +nix-repl> :l file-b.nix +Added 3 variables. +fromB, fromBFails, funcFromB + +nix-repl> :l ./does-not-exist.nix +error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist + +nix-repl> :p fromBFails +error: + … while calling the 'throw' builtin + at «string»:1:18: + 1| fromBFails + | ^ + + error: b + +nix-repl> :doc funcFromB +error: basic_string::substr: __pos (which is 3) > this->size() (which is 0) diff --git a/tests/functional/repl/reload-with-non-existent-file.in b/tests/functional/repl/load-and-reload.in similarity index 70% rename from tests/functional/repl/reload-with-non-existent-file.in rename to tests/functional/repl/load-and-reload.in index 740b3be9a412..a8b94a265bf2 100644 --- a/tests/functional/repl/reload-with-non-existent-file.in +++ b/tests/functional/repl/load-and-reload.in @@ -1,5 +1,7 @@ -:l file-a.nix -:l ./does-not-exist.nix :l file-b.nix +:l ./does-not-exist.nix +:p fromBFails +:doc funcFromB +:l file-a.nix :r fromA + fromB diff --git a/tests/functional/repl/reload-with-non-existent-file.expected b/tests/functional/repl/reload-with-non-existent-file.expected deleted file mode 100644 index e15be6e73085..000000000000 --- a/tests/functional/repl/reload-with-non-existent-file.expected +++ /dev/null @@ -1,24 +0,0 @@ -Nix -Type :? for help. - -nix-repl> :l file-a.nix -Added 1 variables. -fromA - -nix-repl> :l ./does-not-exist.nix -error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist - -nix-repl> :l file-b.nix -Added 1 variables. -fromB - -nix-repl> :r -Loading "file-a.nix"... -Added 1 variables. -fromA -Loading "file-b.nix"... -Added 1 variables. -fromB - -nix-repl> fromA + fromB -3 From 41381f30521b878c39be231773fe240c4d8d1797 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:12:14 +0000 Subject: [PATCH 275/364] build(deps): bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- .github/workflows/ci.yml | 20 ++++++++++---------- .github/workflows/upload-release.yml | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b059853714e5..a4801d756095 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -20,7 +20,7 @@ jobs: with: app-id: ${{ vars.CI_APP_ID }} private-key: ${{ secrets.CI_APP_PRIVATE_KEY }} - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f294211ea74..4e603cb1c085 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: eval: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -40,7 +40,7 @@ jobs: name: pre-commit checks runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: ./.github/actions/install-nix-action with: dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} @@ -87,7 +87,7 @@ jobs: runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -144,7 +144,7 @@ jobs: continue-on-error: true timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -180,7 +180,7 @@ jobs: name: installer test ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Download installer tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -217,7 +217,7 @@ jobs: runs-on: ubuntu-24.04 name: clang-tidy steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -235,14 +235,14 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout nix - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Checkout flake-regressions - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: NixOS/flake-regressions path: flake-regressions - name: Checkout flake-regressions-data - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: NixOS/flake-regressions-data path: flake-regressions/tests @@ -272,7 +272,7 @@ jobs: github.event_name == 'push' && github.ref_name == 'master' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index 9cf01e53c59f..928a088fdbb2 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-24.04 environment: releases steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/install-nix-action with: dogfood: false # Use stable version From a32a9e9906342eca4ca0c16e3c6db622b7261aa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:13:09 +0000 Subject: [PATCH 276/364] build(deps): bump git-hooks-nix in the flake-inputs group Bumps the flake-inputs group with 1 update: [git-hooks-nix](https://github.com/cachix/git-hooks.nix). Updates `git-hooks-nix` from `61ab0e8` to `3bbec39` - [Commits](https://github.com/cachix/git-hooks.nix/compare/61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a...3bbec39bc90eadfa031e6f3b77272f3f60803e39) --- updated-dependencies: - dependency-name: git-hooks-nix dependency-version: 3bbec39bc90eadfa031e6f3b77272f3f60803e39 dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 85432d06e26a..57b5dfbeef05 100644 --- a/flake.lock +++ b/flake.lock @@ -45,11 +45,11 @@ ] }, "locked": { - "lastModified": 1778507602, - "narHash": "sha256-kTwur1wV+01SdqskVMSo6JMEpg71ps3HpbFY2GsflKs=", + "lastModified": 1781733627, + "narHash": "sha256-U3yTuGBnmXvXoQI3qkpfEDsn9RovQPAjN7ndRco+3u0=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a", + "rev": "3bbec39bc90eadfa031e6f3b77272f3f60803e39", "type": "github" }, "original": { From f0f0d950795ef2ea2ac5989ab75b000364c2016d Mon Sep 17 00:00:00 2001 From: Michael Wang <41721295+zwang20@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:13:49 +1000 Subject: [PATCH 277/364] Fix formatting in documentation --- doc/manual/source/command-ref/nix-collect-garbage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee18..07229255e7cd 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,9 +62,9 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--max-freed`](#opt-max-freed) *bytes* - + Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the From a1c93bf07ef4419b6bb19675cdc0459d0a8b6633 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 22 Jun 2026 15:44:13 +0200 Subject: [PATCH 278/364] doc: put system string table next to mechanism description The interjection about cross compilation weakened the connection to the mention that Meson is the source of these system strings. Putting the table right after that mention makes the link more evident. --- doc/manual/source/development/building.md | 31 ++++++++++------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 742170f76c6c..13bcd849e24f 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -199,23 +199,7 @@ Nix uses a string with the following format to identify the *system type* or *pl -[-] ``` -It is set when Nix is compiled for the given system, and based on the output of Meson's [`host_machine` information](https://mesonbuild.com/Reference-manual_builtin_host_machine.html)> - -``` ---[][-] -``` - -When cross-compiling Nix with Meson for local development, you need to specify a [cross-file](https://mesonbuild.com/Cross-compilation.html) using the `--cross-file` option. Cross-files define the target architecture and toolchain. When cross-compiling Nix with Nix, Nixpkgs takes care of this for you. - -In the nix flake we also have some cross-compilation targets available: - -``` -nix build .#nix-everything-riscv64-unknown-linux-gnu -nix build .#nix-everything-armv7l-unknown-linux-gnueabihf -nix build .#nix-everything-armv7l-unknown-linux-gnueabihf -nix build .#nix-everything-x86_64-unknown-freebsd -nix build .#nix-everything-x86_64-w64-mingw32 -``` +It is set when Nix is compiled for the given system, and based on the output of Meson's [`host_machine` information](https://mesonbuild.com/Reference-manual_builtin_host_machine.html). For historic reasons and backward-compatibility, some CPU and OS identifiers are translated as follows: @@ -232,6 +216,19 @@ For historic reasons and backward-compatibility, some CPU and OS identifiers are | `mips` | `big` | `mips` | | `mips64` | `big` | `mips64` | + +When cross-compiling Nix with Meson for local development, you need to specify a [cross-file](https://mesonbuild.com/Cross-compilation.html) using the `--cross-file` option. Cross-files define the target architecture and toolchain. When cross-compiling Nix with Nix, Nixpkgs takes care of this for you. + +In the nix flake we also have some cross-compilation targets available: + +``` +nix build .#nix-everything-riscv64-unknown-linux-gnu +nix build .#nix-everything-armv7l-unknown-linux-gnueabihf +nix build .#nix-everything-armv7l-unknown-linux-gnueabihf +nix build .#nix-everything-x86_64-unknown-freebsd +nix build .#nix-everything-x86_64-w64-mingw32 +``` + ## Compilation environments Nix can be compiled using multiple environments: From 09c8a1b9aac0b041136dd178be6db6666d6fdd75 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 22 Jun 2026 11:13:27 -0500 Subject: [PATCH 279/364] Fix boost format error when diff hook fails Signed-off-by: Lisanna Dettwyler --- src/libstore/unix/build/derivation-builder.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 94bff9c2bc01..3b02dfc4f172 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -137,8 +137,7 @@ static void handleDiffHook( .gid = gid, .chdir = "/"}); if (!statusOk(diffRes.first)) - throw ExecError( - diffRes.first, "diff-hook program %s %2%", PathFmt(diffHook), statusToString(diffRes.first)); + throw ExecError(diffRes.first, "diff-hook program %s %s", PathFmt(diffHook), statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); From e6cf0d2dcab3d8b675b817399f715a3fd5337cb4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 00:27:06 +0300 Subject: [PATCH 280/364] Revert "libexpr: Clear PosTable contents in EvalState::resetFileCache" This reverts commit a091a8100a8587185e579d4cff04381e8e074f12. --- src/libexpr/eval.cc | 1 - .../functional/repl/load-and-reload.expected | 29 ++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 069d7e2c5190..29ecec69ee64 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1189,7 +1189,6 @@ void EvalState::resetFileCache() fileEvalCache->clear(); inputCache->clear(); lookupPathResolved->clear(); - positions.clear(); rootFS->invalidateCache(); } diff --git a/tests/functional/repl/load-and-reload.expected b/tests/functional/repl/load-and-reload.expected index 1642987fe6bc..7f953e28613a 100644 --- a/tests/functional/repl/load-and-reload.expected +++ b/tests/functional/repl/load-and-reload.expected @@ -11,11 +11,32 @@ error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist nix-repl> :p fromBFails error: … while calling the 'throw' builtin - at «string»:1:18: - 1| fromBFails - | ^ + at /path/to/tests/functional/repl/file-b.nix:2:16: + 1| { + 2| fromBFails = throw "b"; + | ^ + 3| fromB = 2; error: b nix-repl> :doc funcFromB -error: basic_string::substr: __pos (which is 3) > this->size() (which is 0) +Function `funcFromB`\ + … defined at /path/to/tests/functional/repl/file-b.nix:7:15 + + +Some documentation. + +nix-repl> :l file-a.nix +Added 1 variables. +fromA + +nix-repl> :r +Loading "file-b.nix"... +Added 3 variables. +fromB, fromBFails, funcFromB +Loading "file-a.nix"... +Added 1 variables. +fromA + +nix-repl> fromA + fromB +3 From d8e330749a23511794b6310d076086d754b21c78 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 01:01:55 +0300 Subject: [PATCH 281/364] Align readline/editline final prompt behavior Aligning these different behaviours seems fraught, so this is like an easier solution. See: https://hydra.nixos.org/build/331681294/log --- src/libcmd/include/nix/cmd/repl-interacter.hh | 4 +++ src/libcmd/repl-interacter.cc | 28 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 7cba481059c9..9b9a03c3e423 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -1,8 +1,10 @@ #pragma once /// @file +#include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" +#include "nix/util/terminal.hh" #include "nix/util/types.hh" #include #include @@ -39,6 +41,8 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { std::filesystem::path historyFile; + bool isInteractive = nix::isTTY(getStandardInput()); + public: ReadlineLikeInteracter(std::filesystem::path historyFile) : historyFile(std::move(historyFile)) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 81240af7f547..5f2417dc1331 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -203,8 +203,25 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT setupSignals(); #endif - char * s = readline(promptForType(promptType)); - Finally doFree([&]() { free(s); }); + + /* Buffer for the non-interactive input. */ + std::string buffer; + const char * s = nullptr; + char * rl = nullptr; + + /* Use plain std::getline for non-interactive mode, which we also use for + testing purposes. readline/editline seem to disagree too much about how + to handle final prompts etc., so it's easier to bypass those. The tests + are mostly about testing the core repl logic, not input handling. */ + if (isInteractive) { + rl = ::readline(promptForType(promptType)); + s = rl; + } else { + s = std::getline(std::cin, buffer) ? buffer.c_str() : nullptr; + } + + Finally doFree([&]() { ::free(rl); }); + #ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); #endif @@ -215,15 +232,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - // editline doesn't echo the input to the output when non-interactive, unlike readline - // this results in a different behavior when running tests. The echoing is - // quite useful for reading the test output, so we add it here. + /* Echo the prompt into the output if run in non-interactive mode, somewhat + for the purposes of characterisation tests. */ if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { -#if !USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. std::cout << promptForType(promptType) << s << std::endl; -#endif } if (!s) From d4248c87a4aac4df91282190abe235d55d80b21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 23 Jun 2026 10:01:53 +0000 Subject: [PATCH 282/364] nix-store --register-validity: fix ENOENT under chroot stores With --store local?root=, registerValidity() passed the logical store path to canonicalisePathMetaData(), which then tried to lstat a path that does not exist on the host filesystem and failed with ENOENT. Use toRealPath() so the chroot-prefixed real path is canonicalised instead. ensureLocalStore() is already required at the end of this function, so hoisting the downcast does not narrow the set of accepted stores. --- src/nix/nix-store/nix-store.cc | 5 +++-- tests/functional/chroot-store.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index e90c7768d9d4..2bb56eb51bca 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -570,6 +570,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs) static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) { + auto localStore = ensureLocalStore(); ValidPathInfos infos; while (1) { @@ -587,7 +588,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) /* !!! races */ if (canonicalise) canonicalisePathMetaData( - store->printStorePath(info->path), + localStore->toRealPath(info->path), {NIX_WHEN_SUPPORT_ACLS(settings.getLocalSettings().ignoredAcls)}); if (!hashGiven) { HashResult hash = hashPath( @@ -601,7 +602,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) } } - ensureLocalStore()->registerValidPaths(infos); + localStore->registerValidPaths(infos); } static void opLoadDB(Strings opFlags, Strings opArgs) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index cbb80c8710ad..d0f41ee87218 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -60,6 +60,15 @@ PATH7=$(nix path-info --store "local://$TEST_ROOT/x%2Bchroot" "$CORRECT_PATH") # Path gets decoded. [[ ! -d "$TEST_ROOT/x%2Bchroot" ]] +# Regression test: `nix-store --register-validity` against a chroot store must +# canonicalise the real (chroot-prefixed) path, not the logical store path +# (which does not exist on the host filesystem). +regPath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-reg +touch "$TEST_ROOT/x/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-reg" +[[ ! -e "$regPath" ]] +(echo "$regPath" && echo && echo 0) | nix-store --store "local?root=$TEST_ROOT/x" --register-validity +nix-store --store "$TEST_ROOT/x" --check-validity "$regPath" + # Ensure store info trusted works with local store nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' From f2c1d45dfc54a263198613c9499c3715aaef4edc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 21:08:37 +0300 Subject: [PATCH 283/364] Fix #15916 The core of the issue is that the trampoline goal reported an arbitrary exitCode, which without --keep-going ends up cancelling goals and we end up dying with an assert in Goal::amDone(). See: https://github.com/NixOS/nix/pull/16042#issuecomment-4760250433 --- .../build/derivation-trampoline-goal.cc | 79 ++++++++++++++++--- tests/functional/meson.build | 1 + .../multiple-outputs-substitute-failure.sh | 28 +++++++ 3 files changed, 97 insertions(+), 11 deletions(-) create mode 100755 tests/functional/multiple-outputs-substitute-failure.sh diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index edf8d1e86ebc..14864ed052f8 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -2,6 +2,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/derivations.hh" +#include +#include + namespace nix { DerivationTrampolineGoal::DerivationTrampolineGoal( @@ -144,11 +147,14 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation }, wantedOutputs.raw); + /* Must have at least one wanted output. This is assumed below. */ + assert(!resolvedWantedOutputs.empty()); + Goals concreteDrvGoals; /* Build this step! */ - auto sharedDrv = make_ref(std::move(drv)); + auto sharedDrv = make_ref(std::move(drv)); for (auto & output : resolvedWantedOutputs) { auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); @@ -157,20 +163,71 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation concreteDrvGoals.insert(std::move(g)); } - // Copy on purpose - co_await await(Goals(concreteDrvGoals)); + co_await await(concreteDrvGoals); trace("outer build done"); - auto & g = *concreteDrvGoals.begin(); - buildResult = g->buildResult; - if (auto * successP = buildResult.tryGetSuccess()) - for (auto & g2 : concreteDrvGoals) - if (auto * successP2 = g2->buildResult.tryGetSuccess()) - for (auto && [x, y] : successP2->builtOutputs) - successP->builtOutputs.insert_or_assign(x, y); + if (nrFailed != 0) { + auto gi = std::ranges::find_if(concreteDrvGoals, [](const GoalPtr & goal) -> bool { + auto exitCode = goal->exitCode; + /* Note that without --keep-going waitees might be cancelled before + we are woken up. */ + return exitCode != ecBusy && exitCode != ecSuccess; + }); + + const Goal * g = gi->get(); + assert(gi != concreteDrvGoals.end() && "expected a failing goal"); + auto exitCode = g->exitCode; + const auto * failure = g->buildResult.tryGetFailure(); + assert(failure && "failing goal does not report a failed build result"); + + /* Report the exit status of *some* failing goal. This might not be strictly + correct, since multiple subgoals can fail independently, but this should be + a good enough heuristic without --keep-going. */ + co_return doneFailure(exitCode, *failure); + } + + SingleDrvOutputs outputs; + + auto successes = std::views::transform(concreteDrvGoals, [](const GoalPtr & a) -> const BuildResult::Success & { + auto * success = a->buildResult.tryGetSuccess(); + assert(success && "goal succeeded, but some waitees do not report a successful status"); + return *success; + }); + + for (const auto & success : successes) + std::ranges::copy(success.builtOutputs, std::inserter(outputs, outputs.end())); + + auto statuses = successes | std::views::transform(&BuildResult::Success::status); + + /* Aggregate the status code. If some outputs we already valid, but we had + to build/substitute the other ones, report it as the smallest common + denominator. */ + auto compareSuccesses = [](auto a, auto b) { + /* This is technically an identity mapping of the underlying values, but + it would be worse to rely on the enum ordering here. */ + auto toPriority = [](auto st) { + using enum BuildResult::Success::Status; + switch (st) { + case Built: + return 0; + case Substituted: + return 1; + case AlreadyValid: + return 2; + case ResolvesToAlreadyValid: + return 3; + default: + unreachable(); + } + }; + return toPriority(a) < toPriority(b); + }; - co_return amDone(g->exitCode); + co_return doneSuccess({ + .status = std::ranges::min(statuses, compareSuccesses), + .builtOutputs = std::move(outputs), + }); } } // namespace nix diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 7d5110ebc4da..8fdaf4ffe018 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -129,6 +129,7 @@ suites = [ 'logging.sh', 'make-content-addressed.sh', 'misc.sh', + 'multiple-outputs-substitute-failure.sh', 'multiple-outputs.sh', 'nar-access.sh', 'nars.sh', diff --git a/tests/functional/multiple-outputs-substitute-failure.sh b/tests/functional/multiple-outputs-substitute-failure.sh new file mode 100755 index 000000000000..fd4a7ec50b15 --- /dev/null +++ b/tests/functional/multiple-outputs-substitute-failure.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# See https://github.com/NixOS/nix/issues/15916 + +source common.sh + +TODO_NixOS # Requires substituting from a local binary cache, enable when we sign paths in NixOS tests +needLocalStore "'--no-require-sigs' can’t be used with the daemon" + +BINARY_CACHE=file://$cacheDir + +readarray -t outPaths < <(nix build -f multiple-outputs.nix 'independent^*' --no-link --print-out-paths) +[[ ${#outPaths[@]} -eq 2 ]] +nix copy --to "$BINARY_CACHE" "${outPaths[@]}" +for p in "${outPaths[@]}"; do + [[ $p == *-second ]] && secondOut=$p +done + +# Corrupt the second output, so that the substitution partially succeeds. +secondNarInfoFile="$cacheDir/$(basename "$secondOut" | cut -c1-32).narinfo" +sed -i 's|^NarHash:.*|NarHash: sha256:0000000000000000000000000000000000000000000000000000|' "$secondNarInfoFile" + +clearStore +clearCacheCache + +# Note that using "^*" matters here. We want all wanted outputs for the same goal. +expect 1 nix build -j 0 -f multiple-outputs.nix "independent^*" --no-link \ + --substituters "$BINARY_CACHE" --no-require-sigs --substitute 2>&1 | grepQuiet "hash mismatch" From dace45aef52d7249b057c1526d1348ea597d2bae Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 24 Jun 2026 21:38:40 +0300 Subject: [PATCH 284/364] libstore: Drop machineName from HookInstance It's now just a local variable. --- .../build/derivation-building-goal.cc | 22 +++++++++---------- .../include/nix/store/build/hook-instance.hh | 9 -------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 81612b4dbc39..5531f984b966 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -655,12 +655,14 @@ Goal::Co DerivationBuildingGoal::buildWithHook( destroyed (e.g., during failure cascades). */ hook->onKillChild = [this]() { worker.childTerminated(this, JobCategory::Build); }; - try { - hook->machineName = readLine(hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the machine name from the build hook"); - throw; - } + std::string machineName = [&hook]() { + try { + return readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw; + } + }(); CommonProto::WriteConn conn{hook->sink}; @@ -699,16 +701,12 @@ Goal::Co DerivationBuildingGoal::buildWithHook( : buildMode == bmCheck ? "checking outputs of '%s'" : "building '%s'", worker.store.printStorePath(drvPath)); - msg += fmt(" on '%s'", hook->machineName); + msg += fmt(" on '%s'", machineName); std::unique_ptr buildLog = std::make_unique( worker.settings.logLines, std::make_unique( - *logger, - lvlInfo, - actBuild, - msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1})); + *logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), machineName, 1, 1})); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); diff --git a/src/libstore/unix/include/nix/store/build/hook-instance.hh b/src/libstore/unix/include/nix/store/build/hook-instance.hh index e53a791d71fc..b3c9fd91385c 100644 --- a/src/libstore/unix/include/nix/store/build/hook-instance.hh +++ b/src/libstore/unix/include/nix/store/build/hook-instance.hh @@ -39,15 +39,6 @@ struct HookInstance */ Pid pid; - /** - * The remote machine on which we're building. - * - * @Invariant When the hook instance is owned by the `Worker`, this - * is the empty string. When it is owned by a `Goal`, this should be - * set. - */ - std::string machineName; - FdSink sink; std::map activities; From 49baeb9b817e15305915f02aed876fefb440b26d Mon Sep 17 00:00:00 2001 From: Yifei Sun Date: Wed, 24 Jun 2026 21:17:44 +0200 Subject: [PATCH 285/364] libfetchers: namespace input fingerprint by scheme --- src/libfetchers/fetchers.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6d7266e09dbb..2cb00fa25308 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -125,6 +125,9 @@ std::optional Input::getFingerprint(Store & store) const auto fingerprint = scheme->getFingerprint(store, *this); + if (fingerprint) + fingerprint = std::string(scheme->schemeName()) + ":" + *fingerprint; + cachedFingerprint = fingerprint; return fingerprint; From f218c7e8ee1b488921e78e0309eee1a22872366b Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 23 Mar 2026 17:04:25 -0400 Subject: [PATCH 286/364] cli: error when remounting store without a private mount namespace Previously, when unshare(CLONE_NEWNS) failed and the store was read-only, Nix warned but still remounted the store writable on the host mount table. This silently affected other processes sharing the namespace. Now it throws an error, since proceeding would mutate shared state. --- src/libstore/local-store.cc | 36 +--------- .../include/nix/util/linux-namespaces.hh | 22 +++--- src/libutil/linux/linux-namespaces.cc | 67 ++++++++++++++++++- src/nix/main.cc | 11 +-- 4 files changed, 83 insertions(+), 53 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4a4ffc988a8..1fb5d20984a3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -37,9 +37,7 @@ #endif #ifdef __linux__ -# include -# include -# include +# include "nix/util/linux-namespaces.hh" #endif #ifdef __CYGWIN__ @@ -649,37 +647,7 @@ void LocalStore::makeStoreWritable() #ifdef __linux__ if (!isRootUser()) return; - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) - throw SysError("getting info about the Nix store mount point"); - - if (stat.f_flag & ST_RDONLY) { - /* In a user namespace, mount flags like `nodev` and `nosuid` are - locked and dropping them causes `EPERM`, so here we translate each - `statvfs` flag to the corresponding `mount` flag individually. */ - unsigned long flags = MS_REMOUNT | MS_BIND; - if (stat.f_flag & ST_NODEV) - flags |= MS_NODEV; - if (stat.f_flag & ST_NOSUID) - flags |= MS_NOSUID; - if (stat.f_flag & ST_NOEXEC) - flags |= MS_NOEXEC; - if (stat.f_flag & ST_NOATIME) - flags |= MS_NOATIME; - if (stat.f_flag & ST_NODIRATIME) - flags |= MS_NODIRATIME; - if (stat.f_flag & ST_RELATIME) - flags |= MS_RELATIME; - if (stat.f_flag & ST_SYNCHRONOUS) - flags |= MS_SYNCHRONOUS; -# ifdef ST_NOSYMFOLLOW - if (stat.f_flag & ST_NOSYMFOLLOW) - flags |= MS_NOSYMFOLLOW; -# endif - if (mount(0, config->realStoreDir.get().c_str(), "none", flags, 0) == -1) - throw SysError("remounting %s writable", PathFmt(config->realStoreDir.get())); - } + remountReadOnlyWritable(config->realStoreDir.get()); #endif } diff --git a/src/libutil/linux/include/nix/util/linux-namespaces.hh b/src/libutil/linux/include/nix/util/linux-namespaces.hh index 8f7ffa8df48d..0b6bef7bcabb 100644 --- a/src/libutil/linux/include/nix/util/linux-namespaces.hh +++ b/src/libutil/linux/include/nix/util/linux-namespaces.hh @@ -1,21 +1,27 @@ #pragma once ///@file -#include - -#include "nix/util/types.hh" +#include namespace nix { /** - * Save the current mount namespace. Ignored if called more than - * once. + * Save the parent mount namespace and enter a private one via + * `unshare(CLONE_NEWNS)`. + */ +void tryEnterPrivateMountNamespace(); + +/** + * Remount `path` writable if its mount is read-only, leaving + * already-writable mounts untouched. This throws if we aren't in a + * private mount namespace, since remounting would leak into the + * host mount table. */ -void saveMountNamespace(); +void remountReadOnlyWritable(const std::filesystem::path & path); /** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. + * Restore the parent mount namespace saved when we entered a private + * one. Ignored if `tryEnterPrivateMountNamespace()` never succeeded. */ void restoreMountNamespace(); diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 26a7479050ad..f0416307d126 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -90,8 +91,11 @@ bool mountAndPidNamespacesSupported() static AutoCloseFD fdSavedMountNamespace; static AutoCloseFD fdSavedRoot; +static bool havePrivateMountNs = false; -void saveMountNamespace() +/* Save the current mount namespace so restoreMountNamespace() can return + to it later. Ignored if called more than once. */ +static void saveMountNamespace() { static std::once_flag done; std::call_once(done, []() { @@ -103,12 +107,69 @@ void saveMountNamespace() }); } +void tryEnterPrivateMountNamespace() +{ + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + havePrivateMountNs = true; + } catch (Error & e) { + debug("failed to set up a private mount namespace: %s", e.message()); + } +} + +void remountReadOnlyWritable(const std::filesystem::path & path) +{ + struct statvfs stat; + if (statvfs(path.c_str(), &stat) != 0) + throw SysError("getting mount info for %s", PathFmt(path)); + + if (!(stat.f_flag & ST_RDONLY)) + return; + + if (!havePrivateMountNs) + throw Error( + "cannot remount %s writable: not in a private mount namespace, " + "so the remount would affect the host mount table. " + "This usually happens inside containers or user namespaces where unshare(CLONE_NEWNS) is not permitted", + PathFmt(path)); + + /* In a user namespace, mount flags like `nodev` and `nosuid` are + locked and dropping them causes `EPERM`, so here we translate each + `statvfs` flag to the corresponding `mount` flag individually. */ + unsigned long flags = MS_REMOUNT | MS_BIND; + if (stat.f_flag & ST_NODEV) + flags |= MS_NODEV; + if (stat.f_flag & ST_NOSUID) + flags |= MS_NOSUID; + if (stat.f_flag & ST_NOEXEC) + flags |= MS_NOEXEC; + if (stat.f_flag & ST_NOATIME) + flags |= MS_NOATIME; + if (stat.f_flag & ST_NODIRATIME) + flags |= MS_NODIRATIME; + if (stat.f_flag & ST_RELATIME) + flags |= MS_RELATIME; + if (stat.f_flag & ST_SYNCHRONOUS) + flags |= MS_SYNCHRONOUS; +#ifdef ST_NOSYMFOLLOW + if (stat.f_flag & ST_NOSYMFOLLOW) + flags |= MS_NOSYMFOLLOW; +#endif + if (mount(0, path.c_str(), "none", flags, 0) == -1) + throw SysError("remounting %s writable", PathFmt(path)); +} + void restoreMountNamespace() { + if (!havePrivateMountNs) + return; + try { auto savedCwd = std::filesystem::current_path(); - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); if (fdSavedRoot) { @@ -120,6 +181,8 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); + + havePrivateMountNs = false; } catch (Error & e) { debug(e.msg()); } diff --git a/src/nix/main.cc b/src/nix/main.cc index df54a14ccb66..963bc07ce274 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -398,15 +398,8 @@ void mainWrapped(int argc, char ** argv) flakeSettings.configureEvalSettings(evalSettings); #ifdef __linux__ - if (isRootUser()) { - try { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - } catch (Error & e) { - warn("failed to set up a private mount namespace: %s", e.msg()); - } - } + if (isRootUser()) + tryEnterPrivateMountNamespace(); #endif Finally f([] { logger->stop(); }); From b06d8e6e8b0db0016fe74938c871cb42e1b2f4a3 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 23 Jun 2026 17:07:26 -0500 Subject: [PATCH 287/364] Set MADV_DONTDUMP for bump allocator This should drastically reduce the time needed to generate a coredump. Resolves #16057 Signed-off-by: Lisanna Dettwyler --- src/libutil/bump-memory-resource.cc | 10 +++++++++- src/libutil/include/nix/util/bump-memory-resource.hh | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libutil/bump-memory-resource.cc b/src/libutil/bump-memory-resource.cc index c892b1652d80..938bd2b84e1b 100644 --- a/src/libutil/bump-memory-resource.cc +++ b/src/libutil/bump-memory-resource.cc @@ -1,4 +1,6 @@ #include "nix/util/bump-memory-resource.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/error.hh" #include "nix/util/file-system.hh" #include "nix/util/alignment.hh" #include "nix/util/logging.hh" @@ -57,7 +59,7 @@ static bool canOvercommit(std::size_t reserveSize) #endif // _WIN32 -BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory_resource * upstream) +BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory_resource * upstream, bool dontDump) : upstreamResource(upstream) { #ifndef _WIN32 @@ -83,6 +85,12 @@ BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory base = p; capacity = reserveSize; + +# ifdef MADV_DONTDUMP + static const bool dumpEverything = getEnv("_NIX_CORE_DUMP_EVERYTHING").value_or("0") == "1"; + if (!dumpEverything && dontDump && ::madvise(p, reserveSize, MADV_DONTDUMP)) + throw SysError("calling madvise"); +# endif #endif } diff --git a/src/libutil/include/nix/util/bump-memory-resource.hh b/src/libutil/include/nix/util/bump-memory-resource.hh index 15b160f35fe8..9ae6be048588 100644 --- a/src/libutil/include/nix/util/bump-memory-resource.hh +++ b/src/libutil/include/nix/util/bump-memory-resource.hh @@ -39,7 +39,8 @@ public: explicit BumpMemoryResource( std::size_t reserveSize = defaultReserveSize, - std::pmr::memory_resource * upstream = std::pmr::new_delete_resource()); + std::pmr::memory_resource * upstream = std::pmr::new_delete_resource(), + bool dontDump = true); BumpMemoryResource(BumpMemoryResource &&) = delete; BumpMemoryResource(const BumpMemoryResource &) = delete; From a8419ef8e770401fd6e6ce4bfeacdbe0d7392b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 25 Jun 2026 10:57:14 -0600 Subject: [PATCH 288/364] libstore: detect Rosetta via runtime file instead of spawning arch getDefaultExtraPlatforms() ran `arch -arch x86_64 /usr/bin/true` to decide whether to add x86_64-darwin to extra-platforms. Because this is the default value of the extra-platforms setting, it runs during static initialization of the global `settings` object, so every libstore linked process on Apple silicon paid roughly 13ms of subprocess startup, even ones that never read extra-platforms (devenv version, hook-should-activate, nix --version). Check for the Rosetta runtime file instead. It is present exactly when Rosetta 2 is installed, which is exactly when the exec probe succeeded, and a stat costs about 0.01ms versus 12.5ms for the subprocess. Assisted-by: Claude Code (Claude Opus 4.8) --- src/libstore/globals.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ea76326bf506..5e62408847bf 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,6 +4,7 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" +#include "nix/util/file-system.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" @@ -254,11 +255,11 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. + // + // The runtime file exists iff Rosetta 2 is installed; checking it avoids + // spawning a subprocess during static initialization of `settings`. if (std::string{NIX_LOCAL_SYSTEM} == "aarch64-darwin" - && runProgram( - RunOptions{.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}) - .first - == 0) + && pathExists("/Library/Apple/usr/libexec/oah/libRosettaRuntime")) extraPlatforms.insert("x86_64-darwin"); #endif From 9205295fbd46e741c598acb4fced5de0920e9067 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 26 Jun 2026 15:03:05 +0300 Subject: [PATCH 289/364] installer: Bail out early if running on macOS < 14.0 --- scripts/install-nix-from-tarball.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index f17e4c2af3b9..73f389b6ff16 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -28,14 +28,15 @@ fi OS="$(uname -s)" -# macOS support for 10.12.6 or higher +# Since nixpkgs 25.11 the minimum deployment target is macOS 14.0 if [ "$OS" = "Darwin" ]; then + # shellcheck disable=SC2034 IFS='.' read -r macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF - if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + if [ "$macos_major" -lt 14 ]; then # patch may not be present; command substitution for simplicity - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 14.0 or higher" exit 1 fi fi From 25dbbaf41697511ff09428b89191b2316fb21f59 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 26 Jun 2026 22:36:12 +0300 Subject: [PATCH 290/364] Fix LocalStoreAccessor::maybeLstat Paths that are not valid store path names (i.e. `foo` or `bar`) can't ever exist in the store, and the correct semantics for it is to return `std::nullopt` instead of throwing. Also noticed a lot of bugs in RemoteFSAccessor, but that's for a later patch. There are also lots of missing overrides of pathExists in SourceAccessor implementations, so I wonder whether that should just be made non-virtual. The evaluator doesn't benefit from potentially optimised pathExists, because MountedSourceAccessor and various other combinators don't propagate them. Plus we should typically still query the lstat to do positive caching (CachingSourceAccessor). See https://github.com/NixOS/nix/issues/16017. --- src/libstore/dummy-store.cc | 3 ++ .../include/nix/store/remote-fs-accessor.hh | 2 + src/libstore/local-fs-store.cc | 37 +++++++++++++++---- src/libstore/remote-fs-accessor.cc | 2 + .../include/nix/util/source-accessor.hh | 7 ++++ tests/functional/flakes/follow-paths.sh | 2 +- tests/functional/lang.sh | 2 +- .../functional/lang/eval-okay-pathexists.nix | 3 ++ 8 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b364c90e160f..6238c8890499 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -69,6 +69,8 @@ class WholeStoreViewAccessor : public SourceAccessor }); if (!res) + /* The accessor is truly empty, i.e. without any file at root so + any subsequent operation with it will fail. */ res = &emptyAccessor; return callback(*res, path); @@ -107,6 +109,7 @@ class WholeStoreViewAccessor : public SourceAccessor DirEntries readDirectory(const CanonPath & path) override { + /* FIXME: Special-case the root directory to read the whole store, not just an empty root. */ return callWithAccessorForPath( path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); } diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index fa7f5fc28052..26fae0d63dd2 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -33,6 +33,8 @@ public: /** * @return nullptr if the store does not contain any object at that path. + * + * @todo This actually doesn't return nullptr, but throws on invalid paths. */ std::shared_ptr accessObject(const StorePath & path); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 5be0e5b0673b..a6fd24bf34e6 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -49,13 +49,33 @@ struct LocalStoreAccessor : SourceAccessor { } - void requireStoreObject(const CanonPath & path) + void requireStoreObject(const StorePath & storePath) { - auto [storePath, rest] = store->toStorePath(store->storeDir + path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); } + static StorePath getStoreObjectPath(const CanonPath & path) + { + /* See special handling of isRoot() in maybeLstat. */ + if (path.isRoot()) + throw BadStorePath("path '%1%' is not a valid store path", path); + return StorePath(*path.begin()); + } + + static std::optional maybeGetStoreObjectPath(const CanonPath & path) + try { + return getStoreObjectPath(path); + } catch (BadStorePath &) { + /* FIXME: Stop using exceptions for control flow. */ + return std::nullopt; + } + + void requireStoreObject(const CanonPath & path) + { + requireStoreObject(getStoreObjectPath(path)); + } + std::optional maybeLstat(const CanonPath & path) override { /* Also allow `path` to point to the entire store, which is @@ -63,7 +83,13 @@ struct LocalStoreAccessor : SourceAccessor if (path.isRoot()) return Stat{.type = tDirectory}; - requireStoreObject(path); + /* Querying existence should not fail for things like + `/nix/store/foo.nix`. The store cannot contain such files (unless + some weird impurities sneak in, but that's UB from nix's PoV). */ + auto maybeStorePath = maybeGetStoreObjectPath(path); + if (!maybeStorePath) + return std::nullopt; + requireStoreObject(*maybeStorePath); return accessor->maybeLstat(path); } @@ -123,11 +149,6 @@ struct LocalStoreAccessor : SourceAccessor { return accessor->getLastModified(); } - - bool pathExists(const CanonPath & path) override - { - return accessor->pathExists(path); - } }; } // namespace diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0129be46eb9b..acf6a4680de4 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -39,6 +39,8 @@ std::optional RemoteFSAccessor::maybeLstat(const CanonPath { if (path.isRoot()) return Stat{.type = tDirectory}; + /* FIXME: Correctly handle invalid names (return nullopt) and don't fail on + non-existent paths. */ auto res = fetch(path); return res.first->maybeLstat(res.second); } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 0c713fcdb518..dcb8f3bc0c6a 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -83,6 +83,13 @@ public: */ virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}); + /** + * @brief Check whether a file exists at @p path. + * + * @todo Consider making this non-virtual, since the evaluator uses + * maybeLstat as an indication that a file exists always (for positive + * caching purposes). + */ virtual bool pathExists(const CanonPath & path); enum Type { diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 143d0c2255e2..39e87348f140 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -131,7 +131,7 @@ EOF git -C "$flakeFollowsA" add flake.nix expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "'flakeB' is too short to be a valid store path" +expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep '/flakeB.*does not exist' # Test relative non-flake inputs. cat > "$flakeFollowsA"/flake.nix <&1 | grepQuiet Hello diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index 022b22feae53..a5a3875da173 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -32,3 +32,6 @@ builtins.pathExists (./lib.nix) && builtins.pathExists ./symlink-resolution/foo/overlays/overlay.nix && builtins.pathExists ./symlink-resolution/broken && builtins.pathExists (builtins.toString ./symlink-resolution/foo/overlays + "/.") +&& builtins.pathExists "${builtins.storeDir}" +&& !builtins.pathExists "${builtins.storeDir}/foo" +&& !builtins.pathExists "${builtins.storeDir}/foo/bar" From 22d3374f8d1220794172ddd29572e0e68240ff44 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 27 Jun 2026 23:40:03 +0300 Subject: [PATCH 291/364] libfetchers/git-utils: Fix #15680 Replace the assertion with a proper error. Also add a test and slightly deduplicate git-utils tests repo creation logic. Also sprinkle some `final` on top while we are at it. --- src/libfetchers-tests/git-utils.cc | 50 ++++++++++++++++++++---------- src/libfetchers/git-utils.cc | 14 ++++++--- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 580769936d41..f02bcdb84f1a 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -1,5 +1,7 @@ #include "nix/fetchers/git-utils.hh" #include "nix/util/file-system.hh" +#include "nix/util/tests/gmock-matchers.hh" + #include #include #include @@ -19,7 +21,7 @@ namespace nix::fetchers { class GitUtilsTest : public ::testing::Test { // We use a single repository for all tests. - std::unique_ptr delTmpDir; + AutoDelete delTmpDir; protected: std::filesystem::path tmpDir; @@ -27,27 +29,19 @@ class GitUtilsTest : public ::testing::Test public: void SetUp() override { - tmpDir = createTempDir(); - delTmpDir = std::make_unique(tmpDir, true); - - // Create the repo with libgit2 - git_libgit2_init(); - git_repository * repo = nullptr; - auto r = git_repository_init(&repo, tmpDir.string().c_str(), 0); - ASSERT_EQ(r, 0); - git_repository_free(repo); + tmpDir = createTempDir() / "test-git-repo"; + GitRepo::openRepo(tmpDir, {.create = true}); + delTmpDir = AutoDelete(tmpDir, true); } void TearDown() override { - // Destroy the AutoDelete, triggering removal - // not AutoDelete::reset(), which would cancel the deletion. - delTmpDir.reset(); + delTmpDir.deletePath(); } ref openRepo() { - return GitRepo::openRepo(tmpDir, {.create = true}); + return GitRepo::openRepo(tmpDir, {.create = false}); } std::string getRepoName() const @@ -117,12 +111,34 @@ TEST_F(GitUtilsTest, sink_hardlink) sink->flush(); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("does not exist")); - ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); - ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + ASSERT_THAT(e.msg(), ::testing::HasSubstr("does not exist")); + ASSERT_THAT(e.msg(), ::testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), ::testing::HasSubstr("foo-1.1/link")); } }; +TEST_F(GitUtilsTest, sink_no_parent_dir) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->createRegularFile(CanonPath("foo/bar"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "boom", /*executable=*/false); + }); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); +} + TEST_F(GitUtilsTest, peel_reference) { // Create a commit in the repo diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index e03503280831..4aa1430e8c70 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -786,7 +786,7 @@ ref GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti * Raw git tree input accessor. */ -struct GitSourceAccessor : SourceAccessor +struct GitSourceAccessor final : SourceAccessor { private: void anchor() override {}; @@ -1063,7 +1063,7 @@ struct GitSourceAccessor : SourceAccessor } }; -struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor +struct GitExportIgnoreSourceAccessor final : CachingFilteringSourceAccessor { private: void anchor() override {}; @@ -1130,7 +1130,7 @@ void GitFileSystemObjectSink::anchor() {} namespace { -struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink +struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink { ref repo; @@ -1210,15 +1210,19 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink void addNode(State & state, const CanonPath & path, Child && child) { - assert(!path.isRoot()); + if (path.isRoot()) + throw Error("cannot create a file at the root of the git repository"); + auto parent = path.parent(); + assert(parent); Directory * cur = &state.root; for (auto & i : *parent) { auto child = std::get_if( &cur->children.emplace(std::string(i), Child{GIT_FILEMODE_TREE, {Directory()}}).first->second.file); - assert(child); + if (!child) + throw Error("parent of '%1%' is not a directory", path.rel()); cur = child; } From 3b0ae6e79ea2663cda93bd954ca57af25666e0c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:12:10 +0000 Subject: [PATCH 292/364] build(deps): bump aws-actions/configure-aws-credentials Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 6.2.0 to 6.2.1. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/e7f100cf4c008499ea8adda475de1042d6975c7b...254c19bd240aabef8777f48595e9d2d7b972184b) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-version: 6.2.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/upload-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index 928a088fdbb2..68af33081089 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -34,7 +34,7 @@ jobs: # get the same uberhack that nix-shell has to support it. echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV" - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 with: role-to-assume: "arn:aws:iam::080433136561:role/nix-release" role-session-name: nix-release-oidc-${{ github.run_id }} From d19f68cf337e1e0666c8a3ecd4b678a484a83ae1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:13:04 +0000 Subject: [PATCH 293/364] build(deps): bump nixpkgs in the flake-inputs group Bumps the flake-inputs group with 1 update: [nixpkgs](https://github.com/NixOS/nixpkgs). Updates `nixpkgs` from `bd0ff2d` to `714a5f8` - [Commits](https://github.com/NixOS/nixpkgs/commits) --- updated-dependencies: - dependency-name: nixpkgs dependency-version: 714a5f8c4ead6b31148d829288440ed033ccc041 dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 57b5dfbeef05..1f946b3532db 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1780902259, - "narHash": "sha256-YMnBf9lk/LYgvqfmSSJuOGigtRs5Lsy26pJHVlR9yMY=", - "rev": "bd0ff2d3eac24699c3664d5966b9ef36f388e2ca", + "lastModified": 1782535326, + "narHash": "sha256-r4TA57SL7nvj1R+GY/FCLwFU48w9IixTQlzBTIYkt8E=", + "rev": "714a5f8c4ead6b31148d829288440ed033ccc041", "type": "tarball", - "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1550.bd0ff2d3eac2/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.3494.714a5f8c4ead/nixexprs.tar.xz" }, "original": { "type": "tarball", From e65765821a584ade9151f573e95eec6ec42bdedb Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Fri, 8 May 2026 11:58:49 +0300 Subject: [PATCH 294/364] libexpr-c: expose `nix_get_derivation` and `nix_value_auto_call_function` These encapsulate internal evaluator machinery that cannot be replicated through the existing attrset/value access API, which makes them the bare minimum necessary surface for consuemers wanting to to traverse and inspect derivation trees. Signed-off-by: NotAShelf Change-Id: I1f6aa9222083068300de22e3f6aad3ac6a6a6964 --- src/libexpr-c/meson.build | 1 + src/libexpr-c/nix_api_eval.cc | 85 +++++++++++++++++++++++++++++++++++ src/libexpr-c/nix_api_expr.h | 45 +++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 src/libexpr-c/nix_api_eval.cc diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 5c6dc6d666ad..fb3a17ff2998 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -30,6 +30,7 @@ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/common') sources = files( + 'nix_api_eval.cc', 'nix_api_expr.cc', 'nix_api_external.cc', 'nix_api_value.cc', diff --git a/src/libexpr-c/nix_api_eval.cc b/src/libexpr-c/nix_api_eval.cc new file mode 100644 index 000000000000..6a451c1027c7 --- /dev/null +++ b/src/libexpr-c/nix_api_eval.cc @@ -0,0 +1,85 @@ +#include + +#include "nix/expr/eval.hh" +#include "nix/expr/get-drvs.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +static const nix::Value & value_in(const nix_value * value) +{ + if (!value) { + throw std::runtime_error("nix_value is null"); + } + if (!value->value || !value->value->isValid()) { + throw std::runtime_error("nix_value is null or uninitialized"); + } + return *value->value; +} + +static nix::Value & value_in(nix_value * value) +{ + if (!value) { + throw std::runtime_error("nix_value is null"); + } + if (!value->value || !value->value->isValid()) { + throw std::runtime_error("nix_value is null or uninitialized"); + } + return *value->value; +} + +static const nix::Bindings * get_bindings_or_null(nix_value * autoArgs) +{ + if (!autoArgs) { + return nullptr; + } + auto & v = value_in(autoArgs); + if (v.type() == nix::nAttrs) { + return v.attrs(); + } + return nullptr; +} + +extern "C" { + +StorePath * +nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value, bool ignoreAssertionFailures) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = value_in(value); + auto maybePkg = nix::getDerivation(state->state, v, ignoreAssertionFailures); + if (!maybePkg) { + return nullptr; + } + nix::StorePath sp = maybePkg->requireDrvPath(); + return new StorePath{std::move(sp)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_value_auto_call_function( + nix_c_context * context, EvalState * state, nix_value * auto_args, nix_value * fn_val, nix_value * result) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & fn = value_in(fn_val); + auto & res = *result->value; + + const nix::Bindings * b = get_bindings_or_null(auto_args); + if (b) { + state->state.autoCallFunction(*b, fn, res); + } else { + state->state.autoCallFunction(nix::Bindings::emptyBindings, fn, res); + } + } + NIXC_CATCH_ERRS +} + +} // extern "C" diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 3623ee076f6f..fc61cb5f39e1 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -342,6 +342,51 @@ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * o /** @} */ // doxygen group GC +/** @addtogroup libexpr_eval + * @ingroup libexpr + * @brief Higher-level evaluation helpers + * @{ + */ + +/** + * @brief Attempt to interpret a Nix value as a derivation. + * + * If the value represents a derivation, returns its drvPath. Returns NULL + * (without setting an error) when the value is not a derivation and the + * caller should recurse into its attributes instead. + * + * Derivation metadata (name, system, outputs, meta) can be queried from + * the value itself using the existing attrset accessors + * (nix_get_attr_byname, nix_get_string, etc.). + * + * @param[out] context Optional, stores error information. Check + * last_err_code to distinguish NULL-from-error from NULL-as-not-derivation. + * @param[in] state The evaluation state. + * @param[in] value The value to inspect. + * @param[in] ignoreAssertionFailures Whether to ignore AssertionErrors + * in the derivation's meta evaluation. + * @return A new StorePath, or NULL. Free with nix_store_path_free(). + */ +StorePath * +nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value, bool ignoreAssertionFailures); + +/** + * @brief Auto-call a function with auto-args (the --arg / --argstr pattern). + * + * This corresponds to nix::EvalState::autoCallFunction. + * + * @param[out] context Optional, stores error information + * @param[in] state The evaluation state. + * @param[in] auto_args An attrset value containing auto-args, or NULL for + * empty. + * @param[in] fn_val The function to call. + * @param[out] result Pre-allocated nix_value to receive the result. + */ +nix_err nix_value_auto_call_function( + nix_c_context * context, EvalState * state, nix_value * auto_args, nix_value * fn_val, nix_value * result); + +/** @} */ // doxygen group libexpr_eval + // cffi end #ifdef __cplusplus } From 9f2bd6c804afa5f39fc9bbfab6e4d0924e0520ea Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Wed, 10 Jun 2026 16:34:46 +0300 Subject: [PATCH 295/364] libexpr-c: document eval helpers without C++ internals; add tests Signed-off-by: NotAShelf Change-Id: Ib3d476dc41ab653f151e00ef95ba3bd56a6a6964 --- src/libexpr-c/nix_api_expr.h | 63 +++++---- src/libexpr-tests/meson.build | 1 + src/libexpr-tests/nix_api_eval.cc | 212 ++++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 23 deletions(-) create mode 100644 src/libexpr-tests/nix_api_eval.cc diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index fc61cb5f39e1..e0a0b4b730db 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -342,45 +342,62 @@ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * o /** @} */ // doxygen group GC -/** @addtogroup libexpr_eval +/** @defgroup libexpr_eval Evaluation * @ingroup libexpr * @brief Higher-level evaluation helpers * @{ */ /** - * @brief Attempt to interpret a Nix value as a derivation. - * - * If the value represents a derivation, returns its drvPath. Returns NULL - * (without setting an error) when the value is not a derivation and the - * caller should recurse into its attributes instead. - * - * Derivation metadata (name, system, outputs, meta) can be queried from - * the value itself using the existing attrset accessors - * (nix_get_attr_byname, nix_get_string, etc.). - * - * @param[out] context Optional, stores error information. Check - * last_err_code to distinguish NULL-from-error from NULL-as-not-derivation. + * @brief Determine whether a Nix value is a derivation and, if so, return its + * store derivation path. + * + * Forces @p value and inspects it. The value is considered a derivation when it + * is an attribute set whose `type` attribute is the string `"derivation"`; in + * that case its `drvPath` attribute is parsed and returned. Otherwise NULL is + * returned without recording an error, which signals that the caller should + * treat @p value as an ordinary attribute set (e.g. recurse into it). + * + * Only the derivation path is returned. Other metadata (`name`, `system`, + * outputs, `meta`, ...) lives on @p value itself and can be read with the + * attribute-set accessors such as nix_get_attr_byname() and nix_get_string(). + * + * @param[out] context Optional, stores error information. On a NULL return, + * inspect the error code via nix_err_code() to tell the two NULL cases apart: + * NIX_OK means @p value is simply not a derivation, any other code means + * inspection failed. See @ref errors. * @param[in] state The evaluation state. - * @param[in] value The value to inspect. - * @param[in] ignoreAssertionFailures Whether to ignore AssertionErrors - * in the derivation's meta evaluation. - * @return A new StorePath, or NULL. Free with nix_store_path_free(). + * @param[in] value The value to inspect. It is forced by this call. + * @param[in] ignoreAssertionFailures If true, an assertion failure raised while + * forcing @p value is treated as "not a derivation" (NULL is returned without + * an error) rather than being reported as an error. + * @return A newly allocated StorePath holding the derivation path, or NULL. + * Free a non-NULL result with nix_store_path_free(). */ StorePath * nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value, bool ignoreAssertionFailures); /** - * @brief Auto-call a function with auto-args (the --arg / --argstr pattern). + * @brief Call a function, drawing its arguments from an attribute set. + * + * Forces @p fn_val and writes its application into @p result: * - * This corresponds to nix::EvalState::autoCallFunction. + * - If @p fn_val is a function that takes a set of named arguments + * (e.g. `{ a, b ? 1 }: ...`), it is called with an attribute set assembled + * from @p auto_args: each named argument is taken from @p auto_args when + * present; an argument absent from @p auto_args falls back to its default; + * an argument that is both absent and has no default is an error. + * - Otherwise @p fn_val is copied into @p result unchanged. This includes any + * non-function value as well as a function that takes a single unnamed + * argument (e.g. `x: ...`), since there are no named arguments to supply. * * @param[out] context Optional, stores error information * @param[in] state The evaluation state. - * @param[in] auto_args An attrset value containing auto-args, or NULL for - * empty. - * @param[in] fn_val The function to call. - * @param[out] result Pre-allocated nix_value to receive the result. + * @param[in] auto_args Attribute set value supplying the named arguments, or + * NULL to supply none. + * @param[in] fn_val The value to call. + * @param[out] result Pre-allocated nix_value that receives the result. + * @return NIX_OK if the call was successful, an error code otherwise. */ nix_err nix_value_auto_call_function( nix_c_context * context, EvalState * state, nix_value * auto_args, nix_value * fn_val, nix_value * result); diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 50d158209ba7..0927ee24b2b2 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -53,6 +53,7 @@ sources = files( 'json.cc', 'lazy-fetcher-attr.cc', 'main.cc', + 'nix_api_eval.cc', 'nix_api_expr.cc', 'nix_api_external.cc', 'nix_api_value.cc', diff --git a/src/libexpr-tests/nix_api_eval.cc b/src/libexpr-tests/nix_api_eval.cc new file mode 100644 index 000000000000..78906c03be9c --- /dev/null +++ b/src/libexpr-tests/nix_api_eval.cc @@ -0,0 +1,212 @@ +#include "nix_api_store.h" +#include "nix_api_util.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" + +#include "nix/expr/tests/nix_api_expr.hh" +#include "nix/util/tests/string_callback.hh" +#include "nix/util/tests/gmock-matchers.hh" + +#include +#include + +namespace nixC { + +// nix_get_derivation + +TEST_F(nix_api_expr_test, nix_get_derivation_returns_drv_path) +{ + auto expr = R"(derivation { name = "myname"; builder = "mybuilder"; system = "mysystem"; })"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + assert_ctx_ok(); + ASSERT_NE(nullptr, drvPath); + + std::string name; + nix_store_path_name(drvPath, OBSERVE_STRING(name)); + EXPECT_THAT(name, ::testing::HasSubstr("myname")); + EXPECT_THAT(name, ::testing::EndsWith(".drv")); + + nix_store_path_free(drvPath); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_non_derivation_returns_null_without_error) +{ + nix_expr_eval_from_string(ctx, state, "{ a = 1; }", ".", value); + assert_ctx_ok(); + + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + // Not a derivation: NULL with no error recorded, so the caller can tell + // this apart from a genuine failure. + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_non_attrset_returns_null_without_error) +{ + nix_expr_eval_from_string(ctx, state, "42", ".", value); + assert_ctx_ok(); + + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_null_value_is_error) +{ + StorePath * drvPath = nix_get_derivation(ctx, state, nullptr, false); + ASSERT_EQ(nullptr, drvPath); + ASSERT_NE(NIX_OK, nix_err_code(ctx)); +} + +// A derivation-shaped attribute set whose `name` throws an assertion only when +// forced. The outer set is already WHNF, so nix_get_derivation is what triggers +// the failure (while reading the name), exercising the assertion handling. +static constexpr const char * ASSERTING_DRV = R"({ type = "derivation"; name = assert false; "myname"; })"; + +TEST_F(nix_api_expr_test, nix_get_derivation_assertion_ignored) +{ + nix_expr_eval_from_string(ctx, state, ASSERTING_DRV, ".", value); + assert_ctx_ok(); + + // With ignoreAssertionFailures = true the assertion is swallowed and the + // value is reported as "not a derivation": NULL with no error. + StorePath * drvPath = nix_get_derivation(ctx, state, value, true); + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_assertion_propagated) +{ + nix_expr_eval_from_string(ctx, state, ASSERTING_DRV, ".", value); + assert_ctx_ok(); + + // With ignoreAssertionFailures = false the assertion surfaces as an error. + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), ::testing::HasSubstr("assert")); +} + +// nix_value_auto_call_function + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_supplies_args) +{ + nix_expr_eval_from_string(ctx, state, "{ a, b }: a + b", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 1; b = 2; }", ".", args); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + assert_ctx_ok(); + + ASSERT_EQ(3, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_uses_defaults) +{ + nix_expr_eval_from_string(ctx, state, "{ a, b ? 10 }: a + b", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 5; }", ".", args); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + assert_ctx_ok(); + + ASSERT_EQ(15, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_null_args_uses_defaults) +{ + nix_expr_eval_from_string(ctx, state, "{ a ? 7 }: a", ".", value); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // NULL auto_args supplies no arguments; every formal must then have a + // default. + nix_value_auto_call_function(ctx, state, nullptr, value, result); + assert_ctx_ok(); + + ASSERT_EQ(7, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_missing_arg_is_error) +{ + nix_expr_eval_from_string(ctx, state, "{ a, b }: a + b", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 1; }", ".", args); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // 'b' has neither a supplied value nor a default. The argument name in the + // message is colorized, so use the ANSI-stripping matcher to assert on it. + nix_value_auto_call_function(ctx, state, args, value, result); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), + ::nix::testing::HasSubstrIgnoreANSIMatcher( + "cannot evaluate a function that has an argument without a value ('b')")); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_non_function_passthrough) +{ + nix_expr_eval_from_string(ctx, state, "42", ".", value); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // A non-function value is returned unchanged. + nix_value_auto_call_function(ctx, state, nullptr, value, result); + assert_ctx_ok(); + + ASSERT_EQ(42, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_single_arg_lambda_passthrough) +{ + nix_expr_eval_from_string(ctx, state, "x: x + 1", ".", value); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // A function taking a single unnamed argument has no named arguments to + // fill, so it is returned unchanged rather than being called. + nix_value_auto_call_function(ctx, state, nullptr, value, result); + assert_ctx_ok(); + + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, result)); + + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_null_fn_is_error) +{ + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, nullptr, nullptr, result); + ASSERT_NE(NIX_OK, nix_err_code(ctx)); + + nix_gc_decref(ctx, result); +} + +} // namespace nixC From a17e200c0f48e0f451e9274b4e34fb88651c50c9 Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Wed, 10 Jun 2026 18:04:18 +0300 Subject: [PATCH 296/364] libexpr-c: deduplicate value validation helpers into internal header Signed-off-by: NotAShelf Change-Id: Ia5e34d12e016a1670b3b64602b13a5256a6a6964 --- src/libexpr-c/nix_api_eval.cc | 30 +++-------------------- src/libexpr-c/nix_api_expr_internal.h | 32 ++++++++++++++++++++++++ src/libexpr-c/nix_api_value.cc | 35 --------------------------- 3 files changed, 35 insertions(+), 62 deletions(-) diff --git a/src/libexpr-c/nix_api_eval.cc b/src/libexpr-c/nix_api_eval.cc index 6a451c1027c7..df3bf3b45072 100644 --- a/src/libexpr-c/nix_api_eval.cc +++ b/src/libexpr-c/nix_api_eval.cc @@ -1,5 +1,3 @@ -#include - #include "nix/expr/eval.hh" #include "nix/expr/get-drvs.hh" @@ -10,34 +8,12 @@ #include "nix_api_util.h" #include "nix_api_util_internal.h" -static const nix::Value & value_in(const nix_value * value) -{ - if (!value) { - throw std::runtime_error("nix_value is null"); - } - if (!value->value || !value->value->isValid()) { - throw std::runtime_error("nix_value is null or uninitialized"); - } - return *value->value; -} - -static nix::Value & value_in(nix_value * value) -{ - if (!value) { - throw std::runtime_error("nix_value is null"); - } - if (!value->value || !value->value->isValid()) { - throw std::runtime_error("nix_value is null or uninitialized"); - } - return *value->value; -} - static const nix::Bindings * get_bindings_or_null(nix_value * autoArgs) { if (!autoArgs) { return nullptr; } - auto & v = value_in(autoArgs); + auto & v = check_value_in(autoArgs); if (v.type() == nix::nAttrs) { return v.attrs(); } @@ -52,7 +28,7 @@ nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value if (context) context->last_err_code = NIX_OK; try { - auto & v = value_in(value); + auto & v = check_value_in(value); auto maybePkg = nix::getDerivation(state->state, v, ignoreAssertionFailures); if (!maybePkg) { return nullptr; @@ -69,7 +45,7 @@ nix_err nix_value_auto_call_function( if (context) context->last_err_code = NIX_OK; try { - auto & fn = value_in(fn_val); + auto & fn = check_value_in(fn_val); auto & res = *result->value; const nix::Bindings * b = get_bindings_or_null(auto_args); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 3f7e4bf1df12..16b32bfa4602 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -2,6 +2,7 @@ #define NIX_API_EXPR_INTERNAL_H #include +#include #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval.hh" @@ -74,4 +75,35 @@ struct nix_realised_string } // extern "C" +// Shared helpers for validating nix_value [in] parameters across libexpr-c translation units. +inline const nix::Value & check_value_not_null(const nix_value * value) +{ + if (!value || !value->value) + throw std::runtime_error("nix_value is null"); + return *value->value; +} + +inline nix::Value & check_value_not_null(nix_value * value) +{ + if (!value || !value->value) + throw std::runtime_error("nix_value is null"); + return *value->value; +} + +inline const nix::Value & check_value_in(const nix_value * value) +{ + auto & v = check_value_not_null(value); + if (!v.isValid()) + throw std::runtime_error("Uninitialized nix_value"); + return v; +} + +inline nix::Value & check_value_in(nix_value * value) +{ + auto & v = check_value_not_null(value); + if (!v.isValid()) + throw std::runtime_error("Uninitialized nix_value"); + return v; +} + #endif // NIX_API_EXPR_INTERNAL_H diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index f15a4aa836a9..eaa5c2e1d535 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -12,41 +12,6 @@ #include "nix_api_store_internal.h" #include "nix_api_value.h" -// Internal helper functions to check [in] and [out] `Value *` parameters -static const nix::Value & check_value_not_null(const nix_value * value) -{ - if (!value) { - throw std::runtime_error("nix_value is null"); - } - return *value->value; -} - -static nix::Value & check_value_not_null(nix_value * value) -{ - if (!value) { - throw std::runtime_error("nix_value is null"); - } - return *value->value; -} - -static const nix::Value & check_value_in(const nix_value * value) -{ - auto & v = check_value_not_null(value); - if (!v.isValid()) { - throw std::runtime_error("Uninitialized nix_value"); - } - return v; -} - -static nix::Value & check_value_in(nix_value * value) -{ - auto & v = check_value_not_null(value); - if (!v.isValid()) { - throw std::runtime_error("Uninitialized nix_value"); - } - return v; -} - static nix::Value & check_value_out(nix_value * value) { auto & v = check_value_not_null(value); From 933f3140b1e73de2b909902b0e251c7da4031607 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 29 Jun 2026 02:39:57 +0300 Subject: [PATCH 297/364] libexpr: Handle lazy paths in builtins.storePath better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building nix.dev manual I ran into an issue with the current handling of lazy paths: … while calling the 'import' builtin at /nix/store/8kpx53qi52yhjai1vdw8zpa95iqa61bv-source/default.nix:167:19: 166| 167| flake = import (outPath + "/flake.nix"); | ^ 168| … while realising the context of a path … while calling the 'storePath' builtin at /nix/store/8kpx53qi52yhjai1vdw8zpa95iqa61bv-source/default.nix:115:15: 114| # If it's already a store path, don't copy it again. 115| builtins.storePath src | ^ 116| else error: path '/nix/store/5fn5lshxlh5zb8w1a747apqlqlbi3j0x-source' is required, but there is no substituter that can build it This also has the benefit of not relying on nix::canonPath(), which is one of the bits of I/O not funneled through the rootFS accessor. The responsible code is in flake-compat [1]. With a path we lack the correct context but it points to a "lazy" path in the store. Funneling I/O into the accessor seems like the correct solution here. [1]: https://github.com/NixOS/flake-compat/blob/5edf11c44bc78a0d334f6334cdaf7d60d732daab/default.nix#L147-L156 --- src/libexpr/primops.cc | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1a837a1b96ff..46ef49a36759 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1961,25 +1961,21 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V .debugThrow(); NixStringContext context; - auto path = - state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'") - .path; - /* Here we are leaving the realm of the rootFS accessor and must actually fetch to the store. - TODO: This could probably get optimised to avoid the fetching altogether to short-circuit when the path - is already mounted on storeFS. */ - state.ensureLazyPathsCopied(context); + SourcePath sourcePath = state.coerceToPath( + pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'"); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true).string()); - if (!state.store->isInStore(path.abs())) - state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); - auto path2 = state.store->toStorePath(path.abs()).first; - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(NixStringContextElem::Opaque{.path = path2}); - v.mkString(path.abs(), context, state.mem); + if (!state.store->isStorePath(sourcePath.path.abs())) + sourcePath = sourcePath.resolveSymlinks(SymlinkResolution::Full); + if (!state.store->isInStore(sourcePath.path.abs())) + state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); + auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) + state.store->ensurePath(storePath); + context.insert(NixStringContextElem::Opaque{.path = storePath}); + v.mkString(sourcePath.path.abs(), context, state.mem); } static RegisterPrimOp primop_storePath({ From 8818381d1f3cbe98857be6416d7747eeeda96892 Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Mon, 29 Jun 2026 07:56:55 +0300 Subject: [PATCH 298/364] libexpr-c: force and validate auto-call arguments Ensure `nix_value_auto_call_function` treats `NULL` auto_args as empty arguments, but forces any provided value and requires it to be an attribute set. Signed-off-by: NotAShelf Change-Id: I17e494abe78f9216dd6346febd784bc26a6a6964 --- src/libexpr-c/nix_api_eval.cc | 20 +++++-------- src/libexpr-tests/nix_api_eval.cc | 47 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/libexpr-c/nix_api_eval.cc b/src/libexpr-c/nix_api_eval.cc index df3bf3b45072..4c7e2e08fc35 100644 --- a/src/libexpr-c/nix_api_eval.cc +++ b/src/libexpr-c/nix_api_eval.cc @@ -8,16 +8,14 @@ #include "nix_api_util.h" #include "nix_api_util_internal.h" -static const nix::Bindings * get_bindings_or_null(nix_value * autoArgs) +static const nix::Bindings & get_bindings_or_empty(nix::EvalState & state, nix_value * autoArgs) { if (!autoArgs) { - return nullptr; + return nix::Bindings::emptyBindings; } auto & v = check_value_in(autoArgs); - if (v.type() == nix::nAttrs) { - return v.attrs(); - } - return nullptr; + state.forceAttrs(v, nix::noPos, "while evaluating automatic function arguments"); + return *v.attrs(); } extern "C" { @@ -46,14 +44,10 @@ nix_err nix_value_auto_call_function( context->last_err_code = NIX_OK; try { auto & fn = check_value_in(fn_val); - auto & res = *result->value; + auto & res = check_value_not_null(result); - const nix::Bindings * b = get_bindings_or_null(auto_args); - if (b) { - state->state.autoCallFunction(*b, fn, res); - } else { - state->state.autoCallFunction(nix::Bindings::emptyBindings, fn, res); - } + auto & b = get_bindings_or_empty(state->state, auto_args); + state->state.autoCallFunction(b, fn, res); } NIXC_CATCH_ERRS } diff --git a/src/libexpr-tests/nix_api_eval.cc b/src/libexpr-tests/nix_api_eval.cc index 78906c03be9c..9d2117678735 100644 --- a/src/libexpr-tests/nix_api_eval.cc +++ b/src/libexpr-tests/nix_api_eval.cc @@ -130,6 +130,53 @@ TEST_F(nix_api_expr_test, nix_value_auto_call_function_uses_defaults) nix_gc_decref(ctx, result); } +TEST_F(nix_api_expr_test, nix_value_auto_call_function_forces_auto_args) +{ + nix_expr_eval_from_string(ctx, state, "{ a }: a + 1", ".", value); + assert_ctx_ok(); + + nix_value * identity = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "x: x", ".", identity); + assert_ctx_ok(); + + nix_value * attrs = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 4; }", ".", attrs); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_init_apply(ctx, args, identity, attrs); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + assert_ctx_ok(); + + ASSERT_EQ(5, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, identity); + nix_gc_decref(ctx, attrs); + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_non_attr_auto_args_is_error) +{ + nix_expr_eval_from_string(ctx, state, "{ a ? 7 }: a", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_init_int(ctx, args, 42); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), ::nix::testing::HasSubstrIgnoreANSIMatcher("expected a set")); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + TEST_F(nix_api_expr_test, nix_value_auto_call_function_null_args_uses_defaults) { nix_expr_eval_from_string(ctx, state, "{ a ? 7 }: a", ".", value); From eab547e3a297b63f81296ce2e78987282b5c5aa1 Mon Sep 17 00:00:00 2001 From: fsagbuya Date: Fri, 26 Jun 2026 15:48:00 +0800 Subject: [PATCH 299/364] libstore: treat `\r\n` as a single line terminator in build logs Build output is processed one character at a time, with '\r' resetting the current line's column and '\n' flushing it. A CRLF line ending thus flushed an empty line, blanking logs from builders that emit '\r\n' (e.g. under Wine). Now '\r' resets the column only when not followed by '\n'. --- src/libstore/build/build-log.cc | 12 ++++++++++-- src/libstore/include/nix/store/build/build-log.hh | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/build-log.cc b/src/libstore/build/build-log.cc index a8fb64fc68bf..920affe12afe 100644 --- a/src/libstore/build/build-log.cc +++ b/src/libstore/build/build-log.cc @@ -10,9 +10,16 @@ BuildLog::BuildLog(size_t maxTailLines, std::unique_ptr act) void BuildLog::operator()(std::string_view data) { - for (auto c : data) + for (auto c : data) { + /* Only let a '\r' reset the column if it isn't followed by '\n', so + "\r\n" acts as a line terminator; defer a char to handle split chunks. */ + if (pendingCR) { + pendingCR = false; + if (c != '\n') + currentLogLinePos = 0; + } if (c == '\r') - currentLogLinePos = 0; + pendingCR = true; else if (c == '\n') flushLine(); else { @@ -20,6 +27,7 @@ void BuildLog::operator()(std::string_view data) currentLogLine.resize(currentLogLinePos + 1); currentLogLine[currentLogLinePos++] = c; } + } } void BuildLog::flush() diff --git a/src/libstore/include/nix/store/build/build-log.hh b/src/libstore/include/nix/store/build/build-log.hh index cdc9125734d1..5dbdf2ba8680 100644 --- a/src/libstore/include/nix/store/build/build-log.hh +++ b/src/libstore/include/nix/store/build/build-log.hh @@ -31,6 +31,8 @@ private: std::string currentLogLine; size_t currentLogLinePos = 0; // to handle carriage return + bool pendingCR = false; // defer '\r' so "\r\n" is treated as a line terminator + void flushLine(); public: From c0a0ce1f49c07a3a4b371f47e83773b05b9b6994 Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Mon, 29 Jun 2026 08:59:02 +0300 Subject: [PATCH 300/364] libexpr-c: correct and simplify eval helper API docs Signed-off-by: NotAShelf Change-Id: Ia8cc4c1b4f78fcacac4334cd8b2d441f6a6a6964 --- src/libexpr-c/nix_api_expr.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index e0a0b4b730db..0326e7becf4a 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -355,8 +355,7 @@ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * o * Forces @p value and inspects it. The value is considered a derivation when it * is an attribute set whose `type` attribute is the string `"derivation"`; in * that case its `drvPath` attribute is parsed and returned. Otherwise NULL is - * returned without recording an error, which signals that the caller should - * treat @p value as an ordinary attribute set (e.g. recurse into it). + * returned without recording an error. * * Only the derivation path is returned. Other metadata (`name`, `system`, * outputs, `meta`, ...) lives on @p value itself and can be read with the @@ -380,7 +379,9 @@ nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value /** * @brief Call a function, drawing its arguments from an attribute set. * - * Forces @p fn_val and writes its application into @p result: + * Forces @p fn_val and writes the application result into @p result. The result + * is not forced; call nix_value_force() to evaluate it before inspecting the + * final value. * * - If @p fn_val is a function that takes a set of named arguments * (e.g. `{ a, b ? 1 }: ...`), it is called with an attribute set assembled From 0db6bc69c1a38bcbbe24a9ff09b963701ea5c917 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2026 17:09:58 +0200 Subject: [PATCH 301/364] SourceAccessor::readFile(sink): Remove default implementation This was defined in terms of the non-virtual string variant of readFile(), which is defined in terms of the sink variant. So this could never work. --- src/libutil/include/nix/util/source-accessor.hh | 3 ++- src/libutil/source-accessor.cc | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 0c713fcdb518..0a325de16856 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -81,7 +81,8 @@ public: * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ - virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}); + virtual void + readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}) = 0; virtual bool pathExists(const CanonPath & path); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 8ae914375418..a7cc160a6dc5 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -71,13 +71,6 @@ std::string SourceAccessor::readFile(const CanonPath & path) return std::move(sink.s); } -void SourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) -{ - auto s = readFile(path); - sizeCallback(s.size()); - sink(s); -} - Hash SourceAccessor::hashPath(const CanonPath & path, PathFilter & filter, HashAlgorithm ha) { HashSink sink(ha); From 05e83636bdf20f75a26d0bf742f9bfbe71299b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 30 Jun 2026 15:48:53 +0000 Subject: [PATCH 302/364] libfetchers: fix race in the Git filesystem object sink The sink built its tree structure from worker threads, after the blob bytes were written. Workers finish in any order, so for "foo" then "foo/bar" the "foo/bar" worker could run first and create "foo" as a directory instead of rejecting it as a file. Hence the flaky GitUtilsTest.sink_no_parent_dir. Reserve each node synchronously, in order; workers only fill in the object ID afterwards. Assisted-by: Claude:unspecified --- src/libfetchers/git-utils.cc | 43 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 4aa1430e8c70..009ddc7c6557 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1232,6 +1232,28 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink cur->children.insert_or_assign(name, std::move(child)); } + /* Set the object ID of a reserved leaf, skipping if it was superseded (id changed) meanwhile. */ + void setNodeOid(State & state, const CanonPath & path, const git_oid & oid, size_t id) + { + auto parent = path.parent(); + assert(parent); + + Directory * cur = &state.root; + for (auto & name : *parent) { + auto i = cur->children.find(std::string(name)); + if (i == cur->children.end()) + return; + auto dir = std::get_if(&i->second.file); + if (!dir) + return; + cur = dir; + } + + auto i = cur->children.find(std::string(*path.baseName())); + if (i != cur->children.end() && i->second.id == id) + i->second.file = oid; + } + void createRegularFile(const CanonPath & path, fun func) override { checkInterrupt(); @@ -1301,6 +1323,10 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink func(*crf); auto id = nextId++; + auto mode = crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB; + + /* Reserve the node now, in order; workers fill the oid later. */ + addNode(*_state.lock(), crf->path, Child{mode, git_oid{}, id}); if (crf->stream) { /* Finish the slow path by creating the blob object synchronously. @@ -1309,10 +1335,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink git_oid oid; if (git_blob_create_from_stream_commit(&oid, crf->stream.release())) throw GitError("creating a blob object for '%s'", path); - addNode( - *_state.lock(), - crf->path, - Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id}); + setNodeOid(*_state.lock(), crf->path, oid, id); return; } @@ -1324,10 +1347,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size())) throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path); - addNode( - *_state.lock(), - crf->path, - Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id}); + setNodeOid(*_state.lock(), crf->path, oid, id); }); } @@ -1341,15 +1361,16 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink void createSymlink(const CanonPath & path, const std::string & target) override { - workers.enqueue([this, path, target]() { + auto id = nextId++; + addNode(*_state.lock(), path, Child{GIT_FILEMODE_LINK, git_oid{}, id}); + workers.enqueue([this, path, target, id]() { auto repo(repoPool.get()); git_oid oid; if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) throw GitError("creating a blob object for tarball symlink member '%s'", path); - auto state(_state.lock()); - addNode(*state, path, Child{GIT_FILEMODE_LINK, oid}); + setNodeOid(*_state.lock(), path, oid, id); }); } From 0d0c3335045ebc7383312265082e93928cd7f77a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Jul 2026 04:08:22 +0300 Subject: [PATCH 303/364] libstore: Fix libcurl thread wakeup with curl >= 8.21 Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 libcurl now swallows events in curl_multi_perform, so there's a chance that we miss a wakeup. This is somewhat reproducible on my machine by doing nix flake prefetch https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz. Another option is to bring back our own wakeup pipe, but that's less portable. Ideally this regression would be fixed in libcurl too... --- src/libstore/filetransfer.cc | 63 ++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d03af296b9b6..c58751e96bf3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -989,6 +989,8 @@ struct curlFileTransfer : public FileTransfer private: bool quitting = false; public: + bool work = false; + void quit() { quitting = true; @@ -1036,12 +1038,14 @@ struct curlFileTransfer : public FileTransfer void stopWorkerThread() { /* Signal the worker thread to exit. */ - state_.lock()->quit(); - wakeupMulti(); + auto state(state_.lock()); + state->quit(); + wakeupMulti(*state); } - void wakeupMulti() + void wakeupMulti(State & state) { + state.work = true; if (auto ec = ::curl_multi_wakeup(curlm.get())) throw curlMultiError(ec); } @@ -1091,25 +1095,12 @@ struct curlFileTransfer : public FileTransfer } } - /* Wait for activity, including wakeup events. */ - long maxSleepTimeMs = items.empty() ? 10000 : 100; - auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() - ? std::max( - 0, - (int) std::chrono::duration_cast( - nextWakeup - std::chrono::steady_clock::now()) - .count()) - : maxSleepTimeMs; - - int numfds = 0; - mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw curlMultiError(mc); - nextWakeup = std::chrono::steady_clock::time_point(); std::vector> incoming; + std::vector> unpause; auto now = std::chrono::steady_clock::now(); + bool haveWork; { auto state(state_.lock()); @@ -1131,7 +1122,9 @@ struct curlFileTransfer : public FileTransfer break; } } + unpause = std::exchange(state->unpause, {}); quit = state->isQuitting(); + haveWork = std::exchange(state->work, false); } for (auto & item : incoming) { @@ -1142,14 +1135,10 @@ struct curlFileTransfer : public FileTransfer items[item->req] = item; } - /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ - auto unpause = [&]() { - auto state(state_.lock()); - auto res = state->unpause; - state->unpause.clear(); - return res; - }(); + if (quit) + break; + /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ for (auto & item : unpause) { /* The transfer might have completed (failed) between it getting enqueued for unpause and by the time the worker thread picked @@ -1159,6 +1148,26 @@ struct curlFileTransfer : public FileTransfer continue; static_cast(*ptr).unpause(); } + + /* Wait for activity, including wakeup events. */ + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + (int) std::chrono::duration_cast( + nextWakeup - std::chrono::steady_clock::now()) + .count()) + : maxSleepTimeMs; + + /* Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 (8.21), + curl_multi_perform seems to swallow queued up events ¯\_(ツ)_/¯. */ + if (haveWork) + sleepTimeMs = 0; + + int numfds = 0; + mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw curlMultiError(mc); } debug("download thread shutting down"); @@ -1195,9 +1204,9 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); item->enqueued = true; /* Now any exceptions should be reported via the callback. */ + wakeupMulti(*state); } - wakeupMulti(); return ItemHandle(item.get_ptr()); } @@ -1217,7 +1226,7 @@ struct curlFileTransfer : public FileTransfer { auto state(state_.lock()); state->unpause.push_back(std::move(item)); - wakeupMulti(); + wakeupMulti(*state); } void unpauseTransfer(ItemHandle handle) override From 4e8ac2e0ede9e3badfca7d7356693c665fa9d57c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 1 Jul 2026 10:19:24 +0000 Subject: [PATCH 304/364] libfetchers: test non-directory parents for symlinks and hardlinks The parent-type check in the Git sink applies to every node kind, not just regular files. Cover symlinks and hardlinks too. Also give the child map a transparent comparator so lookups by string_view no longer allocate a temporary std::string. Assisted-by: Claude:unspecified --- src/libfetchers-tests/git-utils.cc | 40 ++++++++++++++++++++++++++++++ src/libfetchers/git-utils.cc | 10 ++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index f02bcdb84f1a..1d0300287e59 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -139,6 +139,46 @@ TEST_F(GitUtilsTest, sink_no_parent_dir) ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); } +TEST_F(GitUtilsTest, sink_no_parent_dir_symlink) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->createSymlink(CanonPath("foo/bar"), "target"); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); +} + +TEST_F(GitUtilsTest, sink_no_parent_dir_hardlink) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->createHardlink(CanonPath("foo/bar"), CanonPath("foo")); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); +} + TEST_F(GitUtilsTest, peel_reference) { // Create a commit in the repo diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 009ddc7c6557..286d31130e36 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1163,7 +1163,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink /// A directory to be written as a Git tree. struct Directory { - std::map children; + std::map> children; std::optional oid; Child & lookup(const CanonPath & path) @@ -1172,7 +1172,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink auto parent = path.parent(); auto cur = this; for (auto & name : *parent) { - auto i = cur->children.find(std::string(name)); + auto i = cur->children.find(name); if (i == cur->children.end()) throw Error("path '%s' does not exist", path); auto dir = std::get_if(&i->second.file); @@ -1181,7 +1181,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink cur = dir; } - auto i = cur->children.find(std::string(*path.baseName())); + auto i = cur->children.find(*path.baseName()); if (i == cur->children.end()) throw Error("path '%s' does not exist", path); return i->second; @@ -1240,7 +1240,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink Directory * cur = &state.root; for (auto & name : *parent) { - auto i = cur->children.find(std::string(name)); + auto i = cur->children.find(name); if (i == cur->children.end()) return; auto dir = std::get_if(&i->second.file); @@ -1249,7 +1249,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink cur = dir; } - auto i = cur->children.find(std::string(*path.baseName())); + auto i = cur->children.find(*path.baseName()); if (i != cur->children.end() && i->second.id == id) i->second.file = oid; } From 91c9004e0162bce4e4c219fcb4e7813fbce3fc4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 2 Jul 2026 10:32:13 +0000 Subject: [PATCH 305/364] packaging: build aws-c-io with s2n TLS on macOS aws-c-io uses Apple SecureTransport for TLS on macOS. Its handshake calls SSLCreateContext/SecTrust, which route through XPC and abort in a process forked without exec(). The daemon forks a worker per connection without exec, so an S3 substituter that opens a TLS connection during credential resolution (SSO, STS) crashes the worker. Build aws-c-io with s2n instead, which does not touch Apple frameworks. Overriding aws-c-io alone would leave two copies in the closure, so the dependent aws-c-* libraries are rebuilt against it too. Fixes NixOS/nix#15857. --- packaging/aws-c-io-s2n-darwin.patch | 26 ++++++++++++++++++++++++++ packaging/dependencies.nix | 28 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 packaging/aws-c-io-s2n-darwin.patch diff --git a/packaging/aws-c-io-s2n-darwin.patch b/packaging/aws-c-io-s2n-darwin.patch new file mode 100644 index 000000000000..e3d9bf084e08 --- /dev/null +++ b/packaging/aws-c-io-s2n-darwin.patch @@ -0,0 +1,26 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 403bbf6..f095c4d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -100,6 +100,9 @@ elseif (APPLE) + "source/posix/*.c" + "source/darwin/*.c" + ) ++ # nix#15857: use s2n TLS, not fork-unsafe Apple SecureTransport ++ list(REMOVE_ITEM AWS_IO_OS_SRC "${CMAKE_CURRENT_SOURCE_DIR}/source/darwin/secure_transport_tls_channel_handler.c") ++ set(USE_S2N ON) + + find_library(SECURITY_LIB Security) + find_library(NETWORK_LIB Network) +diff --git a/cmake/aws-c-io-config.cmake b/cmake/aws-c-io-config.cmake +index 156e032..d6b222d 100644 +--- a/cmake/aws-c-io-config.cmake ++++ b/cmake/aws-c-io-config.cmake +@@ -1,6 +1,6 @@ + include(CMakeFindDependencyMacro) + +-if (UNIX AND NOT APPLE AND NOT BYO_CRYPTO) ++if (UNIX AND NOT BYO_CRYPTO) + find_dependency(s2n) + endif() + diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index f531b709eec4..5caff664d669 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -56,6 +56,34 @@ scope: { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); }; + # Force the s2n TLS backend in aws-c-io on macOS; Apple SecureTransport is not + # fork-safe and crashes daemon workers (NixOS/nix#15857). Override it across + # the whole aws-c-* stack so one aws-c-io is shared. + aws-crt-cpp = + if !stdenv.hostPlatform.isDarwin then + pkgs.aws-crt-cpp + else + let + aws-c-io = pkgs.aws-c-io.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./aws-c-io-s2n-darwin.patch ]; + }); + aws-c-http = pkgs.aws-c-http.override { inherit aws-c-io; }; + aws-c-auth = pkgs.aws-c-auth.override { inherit aws-c-io aws-c-http; }; + aws-c-event-stream = pkgs.aws-c-event-stream.override { inherit aws-c-io; }; + aws-c-mqtt = pkgs.aws-c-mqtt.override { inherit aws-c-io aws-c-http; }; + aws-c-s3 = pkgs.aws-c-s3.override { inherit aws-c-io aws-c-http aws-c-auth; }; + in + pkgs.aws-crt-cpp.override { + inherit + aws-c-io + aws-c-http + aws-c-auth + aws-c-event-stream + aws-c-mqtt + aws-c-s3 + ; + }; + libgit2 = if lib.versionAtLeast pkgs.libgit2.version "1.9.3" then pkgs.libgit2 From 8c9426502817a493d23ce6e112965544fcb21aed Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Jul 2026 23:20:25 +0300 Subject: [PATCH 306/364] libstore: Make FileTransfer throw Interrupted on interrupted transfers We are now issuing many more requests in queryMissing concurrently, without doing checkInterrupt() when enqueueing downloads (arguably we shouldn't). The exception ignoring behavior of querySubstitutablePathInfosAsync, where it swallows and prints exceptions derived from Error & doesn't help - we end up with a wall of text like: error: download of 'https://cache.nixos.org/b2s9lqzdi2pg80j0lawgw8ndian3k9mn.narinfo' was interrupted error: download of 'https://cache.nixos.org/5h2h4brfrgr89gvh1l93mh63mm0h5r6s.narinfo' was interrupted error: download of 'https://cache.nixos.org/azz5rixma8na4h9s75406w94yjsk2b4k.narinfo' was interrupted error: download of 'https://cache.nixos.org/xdlfnymy64jxfhnj19sjvdvmbbmyn468.narinfo' was interrupted error: download of 'https://cache.nixos.org/q8sra2q3n8gvalzbfq48jvv18xjsg2gb.narinfo' was interrupted error: download of 'https://cache.nixos.org/pzmm2bczb5gb1nvgv5xp6ims9iqcc4af.narinfo' was interrupted error: download of 'https://cache.nixos.org/s3slcivizcjkppyvf5drlbx0b8dxafg9.narinfo' was interrupted Rethrowing Interrupted instead of FileTransferError on proper interrupts seems correct to me. And since it doesn't inherit from Error (but rather BaseError) it escapes out of catch (Error &) blocks and properly unwinds the stack eagerly. Also add a new `Cancelled` exception type to reflect cases (not very useful for now) to indicate that a filetransfer was cancelled without a global interrupt. In the future we should add per-transfer cancellation mechanisms for this purpose. --- src/libstore/filetransfer.cc | 34 ++++++++++++------- .../include/nix/store/filetransfer.hh | 2 +- src/libutil/include/nix/util/signals.hh | 1 + src/libutil/unix/signals.cc | 2 ++ src/libutil/util.cc | 4 +++ src/libutil/windows/signals.cc | 2 ++ 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c58751e96bf3..6e9f2256eaa0 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -294,12 +294,7 @@ struct curlFileTransfer : public FileTransfer } try { if (!done && enqueued) - fail(FileTransferError( - Interrupted, - {}, - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.displayUri())); + failInterruptedOrCancelled(); } catch (...) { ignoreExceptionInDestructor(); } @@ -328,6 +323,18 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward(e))); } + void failInterruptedOrCancelled() + { + HintFmt fmt("%s of '%s' was interrupted", Uncolored(request.noun()), request.displayUri()); + + /* Technically, we don't really have per-transfer cancellation currently, + but it's nice to distinguish between the two in the future. */ + if (getInterrupted()) + fail(nix::Interrupted(std::move(fmt))); + else + fail(nix::Cancelled(std::move(fmt))); + } + LambdaSink finalSink; std::optional errorSink; @@ -872,13 +879,14 @@ struct curlFileTransfer : public FileTransfer std::optional response; if (errorSink) response = std::move(errorSink->s); - auto exc = code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError( - Interrupted, - std::move(response), - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.displayUri()) - : httpStatus != 0 + + /* TODO: Also support per-transfer cancellations. */ + if (code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()) { + failInterruptedOrCancelled(); + return; + } + + auto exc = httpStatus != 0 ? FileTransferError( err, std::move(response), diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 04980c9ac465..774a4e1403b3 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -470,7 +470,7 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); - enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; /** diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 4f9d9516bbf0..5124ebaccafd 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -43,6 +43,7 @@ inline void checkInterrupt(); * @note Never will happen on Windows */ MakeError(Interrupted, BaseError); +MakeError(Cancelled, BaseError); struct InterruptCallback { diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index d1de7842aaaf..38030a77ddfd 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -11,6 +11,8 @@ namespace nix { void Interrupted::anchor() {} +void Cancelled::anchor() {} + std::atomic unix::_isInterrupted = false; thread_local std::function unix::interruptCheck; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f5f427734eb..e6b2fffa25fa 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -249,6 +249,10 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl) throw; } catch (const Interrupted & e) { throw; + } catch (const Cancelled & e) { + /* Morally the same as Interrupted, just not triggered by a user but some other + cancellation. */ + throw; } catch (Error & e) { printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { diff --git a/src/libutil/windows/signals.cc b/src/libutil/windows/signals.cc index ca4900313fc5..40fe5eeb63c4 100644 --- a/src/libutil/windows/signals.cc +++ b/src/libutil/windows/signals.cc @@ -4,4 +4,6 @@ namespace nix { void Interrupted::anchor() {} +void Cancelled::anchor() {} + } // namespace nix From dc8fac0b68168a4c9eb9d491f7abe92205335703 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Jul 2026 23:20:36 +0300 Subject: [PATCH 307/364] libutil: Special-case forEachAsync for one awaitable, document current lack of cancellation We don't need to do the whole async_initiate dance for just one awaitable. --- src/libutil/include/nix/util/async.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/include/nix/util/async.hh b/src/libutil/include/nix/util/async.hh index 3f6be9b48cc7..952cdb5610f8 100644 --- a/src/libutil/include/nix/util/async.hh +++ b/src/libutil/include/nix/util/async.hh @@ -95,10 +95,14 @@ asio::awaitable forEachAsync(Range && range, const F & f) auto pending = std::ranges::size(range); if (pending == 0) co_return; + else if (pending == 1) + co_return co_await f(*range.begin()); auto executor = co_await asio::this_coro::executor; std::exception_ptr err; + /* TODO: Handle cancellation on first error. Not very useful for now since we + do all-or-nothing cancellation typically. */ co_await asio::async_initiate( [&](auto handler) { auto h = std::make_shared(std::move(handler)); From db08619e33d7f1cc8a0fe0004b084adbd9a3769d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 4 Jul 2026 18:39:19 +0300 Subject: [PATCH 308/364] release notes: 2.35.0 Co-authored-by: Lisanna Dettwyler --- doc/manual/rl-next/async-post-build-hook.md | 9 - .../aws-sts-webidentity-region-fallback.md | 11 - doc/manual/rl-next/beta-installer | 29 -- doc/manual/rl-next/build-trace-rework.md | 96 ---- doc/manual/rl-next/closure-gc.md | 8 - doc/manual/rl-next/delete-keep.md | 8 - .../rl-next/filetransfer-retry-backoff.md | 27 -- doc/manual/rl-next/fix-primop-eval-state.md | 10 - doc/manual/rl-next/flake-check-out-links.md | 7 - .../rl-next/flake-check-print-output-paths.md | 7 - doc/manual/rl-next/freebsd-sandboxing.md | 10 - doc/manual/rl-next/getflake-path.md | 6 - doc/manual/rl-next/git-url-scp.md | 30 -- .../github-fetcher-param-validation.md | 7 - .../rl-next/hash-self-reference-positions.md | 12 - doc/manual/rl-next/http3.md | 25 -- doc/manual/rl-next/mimalloc.md | 15 - .../s3-credential-chain-web-identity.md | 26 -- doc/manual/rl-next/seccomp-block-listxattr.md | 10 - .../rl-next/store-config-get-state-dir.md | 36 -- doc/manual/rl-next/zstd-multiframe.md | 18 - doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/release-notes/rl-2.35.md | 414 ++++++++++++++++++ 23 files changed, 415 insertions(+), 407 deletions(-) delete mode 100644 doc/manual/rl-next/async-post-build-hook.md delete mode 100644 doc/manual/rl-next/aws-sts-webidentity-region-fallback.md delete mode 100644 doc/manual/rl-next/beta-installer delete mode 100644 doc/manual/rl-next/build-trace-rework.md delete mode 100644 doc/manual/rl-next/closure-gc.md delete mode 100644 doc/manual/rl-next/delete-keep.md delete mode 100644 doc/manual/rl-next/filetransfer-retry-backoff.md delete mode 100644 doc/manual/rl-next/fix-primop-eval-state.md delete mode 100644 doc/manual/rl-next/flake-check-out-links.md delete mode 100644 doc/manual/rl-next/flake-check-print-output-paths.md delete mode 100644 doc/manual/rl-next/freebsd-sandboxing.md delete mode 100644 doc/manual/rl-next/getflake-path.md delete mode 100644 doc/manual/rl-next/git-url-scp.md delete mode 100644 doc/manual/rl-next/github-fetcher-param-validation.md delete mode 100644 doc/manual/rl-next/hash-self-reference-positions.md delete mode 100644 doc/manual/rl-next/http3.md delete mode 100644 doc/manual/rl-next/mimalloc.md delete mode 100644 doc/manual/rl-next/s3-credential-chain-web-identity.md delete mode 100644 doc/manual/rl-next/seccomp-block-listxattr.md delete mode 100644 doc/manual/rl-next/store-config-get-state-dir.md delete mode 100644 doc/manual/rl-next/zstd-multiframe.md create mode 100644 doc/manual/source/release-notes/rl-2.35.md diff --git a/doc/manual/rl-next/async-post-build-hook.md b/doc/manual/rl-next/async-post-build-hook.md deleted file mode 100644 index ea061f14ae9d..000000000000 --- a/doc/manual/rl-next/async-post-build-hook.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: Make post-build-hook asynchronous -prs: [15451] -issues: [15406] ---- - -This change makes the `post-build-hook` run asynchronously but still as part of the goal. -This retains the current behavior that a waiting goal will not start until the `post-build-hook` of the goal it is waiting on completes. -However, multiple `post-build-hook`s can now run concurrently just as multiple goals can run concurrently. diff --git a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md deleted file mode 100644 index 5034c65438cb..000000000000 --- a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -synopsis: S3 substituters fall back to the URL's region for STS WebIdentity auth -prs: [15594] ---- - -When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, -GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL -as a fallback for the STS endpoint region if neither `AWS_REGION` nor -`AWS_DEFAULT_REGION` is set. Previously, IRSA setups that exported -`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail -with a misleading "IMDS provider" error. diff --git a/doc/manual/rl-next/beta-installer b/doc/manual/rl-next/beta-installer deleted file mode 100644 index b02564d95d2d..000000000000 --- a/doc/manual/rl-next/beta-installer +++ /dev/null @@ -1,29 +0,0 @@ ---- -synopsis: "Rust nix-installer in beta" -prs: [] ---- - -The Rust-based rewrite of the Nix installer is now in beta. -We'd love help testing it out! - -To test out the new installer, run: -``` -curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install -``` - -This installer can be run even when you have an existing, script-based Nix installation without any adjustments. - -This new installer also comes with the ability to uninstall your Nix installation; run: -``` -/nix/nix-installer uninstall -``` - -This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation). - -This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems. -Thanks to Determinate Systems for all the investment they've put into the installer. - -Source for the installer is in https://github.com/NixOS/nix-installer. -Report any issues in that repo. - -For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action. diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md deleted file mode 100644 index 61b6bc51b1c0..000000000000 --- a/doc/manual/rl-next/build-trace-rework.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -synopsis: "Content-addressed derivations: realisations keyed by store path instead of hash modulo" -issues: [11897] -prs: [12464] ---- - -The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. This affects the **binary cache protocol** and the **wire protocols**. - -### What changed - -#### Build trace format - -Previously, a build trace entry (realisation) was keyed by the **hash modulo** of the derivation. -A SHA-256 hash computed via the complex "derivation hash modulo" algorithm. -This required implementations to understand ATerm serialisation and the full derivation hashing scheme just to look up or store build results. - -Now, build trace entries are keyed by the **regular derivation store path** plus the output name. For example, instead of: - -``` -sha256:ba7816bf8f01...!out -``` - -The key is now: - -``` -/nix/store/abc...-foo.drv^out -``` - -This is simpler, more intuitive, and means that third-party tools implementing CA derivation support (e.g., Hydra) -no longer need to implement the derivation hash modulo algorithm. - -#### Build trace usage - -Previously the build trace contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. -Now, it only contains entries for resolved derivations. -For now, unresolved derivations will be resolved from these underlying build trace entries. -This is slower, but avoids a bunch of correctness issues. - -### Binary cache protocol - -- The directory for build traces moved from `realisations/` to `build-trace-v2/`. -- File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. -- The JSON format of build trace entries is now split into `key` and `value` objects: - ```json - { - "key": { - "drvPath": "abc...-foo.drv", - "outputName": "out" - }, - "value": { - "outPath": "xyz...-foo", - "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] - } - } - ``` - Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. -- The deprecated `dependentRealisations` field has been removed. - -Existing binary caches will need to be re-populated with the new format for CA derivation build traces. -Old build traces at the previous URLs are simply abandoned. -Non-CA builds are unaffected. - -### Wire protocols - -- **Worker protocol**: - A new feature flag `realisation-with-path-not-hash` is negotiated during the handshake. - Clients and daemons that both support this feature use the new binary serialisation for `DrvOutput`, `UnkeyedRealisation`, and related types. - Fallback to older protocol versions gracefully degrades (realisations are unavailable). -- **Serve protocol**: - Bumped from 2.7 to 2.8 with native serialisers for the new types. - Fallback to older protocol versions gracefully degrades in the same way. - -Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. -For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. - -### Local Store SQLite schema - -The build trace entries no longer have any foreign key store objects in the store. -This is because we will need to remember the build trace entries for resolved derivations we may have deleted, otherwise we will effectively forget outputs resolved derivations we do have on disk. -GC for build trace will be implemented later --- there is no single correct choice (there is no closure property) so it will be a question of what policies users want. - -### Structured signatures - -[Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. -`nix path-info --json --json-format 3` opts into the new version for this command. -JSON parsing accepts both the old string format and new structured format for backwards compatibility. - -### Impact - -- **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. -- **Binary cache operators**: - Binary caches serving CA derivation build traces will need to be repopulated. - Existing NARs and narinfo files are unaffected. -- **Tool authors**: - Implementations interfacing with the CA derivations protocol are simplified. - The derivation hash modulo algorithm is no longer required to form build trace keys. diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md deleted file mode 100644 index 945a78dc9f8c..000000000000 --- a/doc/manual/rl-next/closure-gc.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" -issues: 7239 -prs: [15236, 15727] ---- - -`nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. -The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. diff --git a/doc/manual/rl-next/delete-keep.md b/doc/manual/rl-next/delete-keep.md deleted file mode 100644 index 0332e0e3933e..000000000000 --- a/doc/manual/rl-next/delete-keep.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands" -prs: [15776] ---- - -Setting `keep-derivations = true` and trying to delete a derivation with realised outputs would previously fail. -Same with `keep-outputs = true` and trying to delete an output that still has derivers. -These options no longer affect the deletion commands, and are now documented as such. diff --git a/doc/manual/rl-next/filetransfer-retry-backoff.md b/doc/manual/rl-next/filetransfer-retry-backoff.md deleted file mode 100644 index 0e26ccd54a8c..000000000000 --- a/doc/manual/rl-next/filetransfer-retry-backoff.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -synopsis: Configurable file-transfer retry backoff with full jitter and Retry-After support -issues: [15419, 15023] -prs: [15449] ---- - -File transfer retries (downloads and uploads) now use AWS-style "full jitter" -exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), -and honor the `Retry-After` response header. Retry timing is configurable via -new `nix.conf` settings: - -- `filetransfer-retry-delay` (default 100ms): base delay for transient errors -- `filetransfer-retry-delay-rate-limited` (default 5000ms): base delay for 429/503 -- `filetransfer-retry-max-delay` (default 60000ms): per-attempt delay ceiling -- `filetransfer-retry-jitter` (default true): enable full jitter - -The existing `download-attempts` setting has been renamed to -`filetransfer-retry-attempts` to reflect that it applies to uploads as well as -downloads. The old name remains as an alias for backwards compatibility. - -Per-substituter overrides are available as store URL parameters -(`retry-delay`, `retry-delay-rate-limited`, `retry-max-delay`, -`retry-attempts`), e.g. `s3://my-cache?retry-attempts=8`. - -This addresses thundering-herd scenarios where many CI jobs hit the same -S3 prefix and receive 503 SlowDown; previously the retry window for 503 -was only ~4 seconds. diff --git a/doc/manual/rl-next/fix-primop-eval-state.md b/doc/manual/rl-next/fix-primop-eval-state.md deleted file mode 100644 index 1a3bc287538f..000000000000 --- a/doc/manual/rl-next/fix-primop-eval-state.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "C API: Fix `EvalState` pointer passed to primop callbacks" -prs: [15300, 15383] ---- - -The `EvalState *` passed to C API primop callbacks was incorrectly pointing to -the internal `nix::EvalState` rather than the C API wrapper struct. This caused -a segfault when the callback used the pointer with C API functions such as -`nix_alloc_value()`. The same issue affected `printValueAsJSON` and -`printValueAsXML` callbacks on external values. diff --git a/doc/manual/rl-next/flake-check-out-links.md b/doc/manual/rl-next/flake-check-out-links.md deleted file mode 100644 index eb369a629c3f..000000000000 --- a/doc/manual/rl-next/flake-check-out-links.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: nix flake check now supports --out-link -prs: [15476] -issues: [13470] ---- - -`nix flake check` now supports the flag `--out-link`, defaulting to not creating out links if the flag is not specified. diff --git a/doc/manual/rl-next/flake-check-print-output-paths.md b/doc/manual/rl-next/flake-check-print-output-paths.md deleted file mode 100644 index 5e0fd35b6737..000000000000 --- a/doc/manual/rl-next/flake-check-print-output-paths.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: nix flake check now supports --print-out-paths -prs: [15476] -issues: [13470] ---- - -`nix flake check` now supports the flag `--print-out-paths`. diff --git a/doc/manual/rl-next/freebsd-sandboxing.md b/doc/manual/rl-next/freebsd-sandboxing.md deleted file mode 100644 index 2c2d214494d9..000000000000 --- a/doc/manual/rl-next/freebsd-sandboxing.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: Enable FreeBSD sandboxing, add `x86_64-freebsd` to installer -prs: [15673, 13281, 9968] ---- - -A FreeBSD build has been added to the traditional installer script, with sandboxing enabled. -The beta installer is not yet supported. - -FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages -and performing other tasks expected of Nix on Linux. diff --git a/doc/manual/rl-next/getflake-path.md b/doc/manual/rl-next/getflake-path.md deleted file mode 100644 index 2360fe7693e0..000000000000 --- a/doc/manual/rl-next/getflake-path.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: "`builtins.getFlake` now supports path values" -prs: [15290] ---- - -`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref. diff --git a/doc/manual/rl-next/git-url-scp.md b/doc/manual/rl-next/git-url-scp.md deleted file mode 100644 index b1125f7b32a7..000000000000 --- a/doc/manual/rl-next/git-url-scp.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -synopsis: Support SCP-like URLs in fetchGit and type = "git" flake inputs -prs: [14863] -issues: [14852, 14867] ---- - -Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially -restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: - -```nix -builtins.fetchGit "host:/absolute/path/to/repo" -``` - -Nix also passes through the tilde (for home directories) verbatim: - -```nix -builtins.fetchGit "host:~/relative/to/home" -``` - -IPv6 addresses also supported when bracketed: - -```nix -builtins.fetchGit "user@[::1]:~/relative/to/home" -``` - -`builtins.fetchTree` also supports this syntax now: - -```nix -builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } -``` diff --git a/doc/manual/rl-next/github-fetcher-param-validation.md b/doc/manual/rl-next/github-fetcher-param-validation.md deleted file mode 100644 index 2fb430ae4b24..000000000000 --- a/doc/manual/rl-next/github-fetcher-param-validation.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: GitHub fetcher now validates URL parameters -prs: [15331] -issues: [15304] ---- - -The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. diff --git a/doc/manual/rl-next/hash-self-reference-positions.md b/doc/manual/rl-next/hash-self-reference-positions.md deleted file mode 100644 index 0a7193b3a497..000000000000 --- a/doc/manual/rl-next/hash-self-reference-positions.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: "Fix hash collision between store paths with self-references and their zeroed-out equivalents" -issues: [15837] -prs: [15931] ---- - -When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. -The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. - -This release restores hashing the positions of self-references. -As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. -This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. diff --git a/doc/manual/rl-next/http3.md b/doc/manual/rl-next/http3.md deleted file mode 100644 index 1f0a31227e6f..000000000000 --- a/doc/manual/rl-next/http3.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -synopsis: HTTP/3 (QUIC) support ---- - -Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 -(QUIC), controlled by a new -[`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by -default). When enabled, Nix requests HTTP/3 and transparently falls back to -HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. The setting only -takes effect when linked against a libcurl built with HTTP/3 support, otherwise -it is ignored and Nix keeps using HTTP/2 without warning or error. - -Enable it with: - -``` -nix.conf: http3 = true -CLI: --http3 -``` - -Or disable with: - -``` -nix.conf: http3 = false -CLI: --no-http3 -``` diff --git a/doc/manual/rl-next/mimalloc.md b/doc/manual/rl-next/mimalloc.md deleted file mode 100644 index dfcde0cab03a..000000000000 --- a/doc/manual/rl-next/mimalloc.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -synopsis: "Link mimalloc for faster evaluation" -prs: [15596] ---- - -The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) -by default on non-Windows platforms, replacing glibc's malloc for all -non-GC allocations. - -This yields a **5–12% wall-clock improvement** on evaluation workloads, -ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS -configurations. - -The allocator can be disabled at build time with `-Dmimalloc=disabled` -or by passing `withMimalloc = false` to the Nix package. diff --git a/doc/manual/rl-next/s3-credential-chain-web-identity.md b/doc/manual/rl-next/s3-credential-chain-web-identity.md deleted file mode 100644 index 4dfece0e3b7d..000000000000 --- a/doc/manual/rl-next/s3-credential-chain-web-identity.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -synopsis: "S3: restore STS WebIdentity and ECS container credential providers" -prs: [15507] ---- - -Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a -custom chain built on `aws-c-auth`. That chain omitted two providers, -breaking S3 binary cache access in container workloads: - -- **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, - `AWS_ROLE_SESSION_NAME`) — used by EKS IRSA, GitHub Actions OIDC, and - any `sts:AssumeRoleWithWebIdentity` federation. -- **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, - `AWS_CONTAINER_CREDENTIALS_FULL_URI`) — used by ECS tasks and EKS Pod - Identity. - -The typical symptom was a misleading IMDS error -(`Valid credentials could not be sourced by the IMDS provider`), because -IMDS is the last provider tried after the correct one was skipped. - -Both providers are now part of the chain, ordered to match the -pre-2.33 `DefaultAWSCredentialsProviderChain`: -`Environment → SSO → Profile → STS WebIdentity → (ECS | IMDS)`. -As in both the old and new AWS SDK default chains, ECS and IMDS are -mutually exclusive: when container credential environment variables are -set, IMDS is skipped. diff --git a/doc/manual/rl-next/seccomp-block-listxattr.md b/doc/manual/rl-next/seccomp-block-listxattr.md deleted file mode 100644 index 2455ed4f825f..000000000000 --- a/doc/manual/rl-next/seccomp-block-listxattr.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "Linux sandbox: also block `listxattr` syscalls" -prs: [15743] ---- - -The Linux sandbox now also returns `ENOTSUP` for `listxattr`, -`llistxattr` and `flistxattr`, matching the existing treatment of -`getxattr`/`setxattr`/`removexattr`. This prevents host xattrs (e.g. -`security.selinux`) from leaking into builds and fixes tools such as -`mkfs.ubifs` that probe xattr support via `listxattr`. diff --git a/doc/manual/rl-next/store-config-get-state-dir.md b/doc/manual/rl-next/store-config-get-state-dir.md deleted file mode 100644 index a789bdf83511..000000000000 --- a/doc/manual/rl-next/store-config-get-state-dir.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -synopsis: "Improve daemon socket path logic for chroot stores" -prs: [15429] ---- - -The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). -This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. - -Example: - -```bash -nix-daemon --store /foo/bar -``` - -will now use a socket at: -``` -/foo/bar/nix/var/nix/daemon-socket/socket -``` -instead of -``` -$NIX_STATE_DIR/daemon-socket/socket -``` - -Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: - -- When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: - - ```bash - nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" - ``` - -- When connecting as a client put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): - - ``` - unix://$NIX_STATE_DIR/daemon-socket/socket - ``` diff --git a/doc/manual/rl-next/zstd-multiframe.md b/doc/manual/rl-next/zstd-multiframe.md deleted file mode 100644 index 6f5d9875df74..000000000000 --- a/doc/manual/rl-next/zstd-multiframe.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -synopsis: zstd compression now emits multi-frame output and uses less memory -prs: [15550] ---- - -zstd-compressed NARs are now written as a sequence of independent 16 MiB -frames instead of a single large frame. This lays the groundwork for -parallel decompression in a future release without requiring caches to be -repopulated, and significantly lowers peak memory use during compression -(e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). - -The output remains standard zstd and is decoded unchanged by existing Nix -binaries and the `zstd` CLI; compression ratio is effectively unchanged. - -Per-frame compression now uses up to 4 worker threads. For zstd this is the -new default: the `parallel-compression` store setting defaults to `true` when -`compression=zstd` (it remains `false` for `xz`). Set -`?parallel-compression=false` to opt out. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index a12e84becc35..ae77759f5516 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -159,6 +159,7 @@ - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.35 (2026-06-22)](release-notes/rl-2.35.md) - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md new file mode 100644 index 000000000000..a712b6385627 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -0,0 +1,414 @@ +# Release 2.35.0 (2026-06-22) + +## Highlights + +- Sources are copied to the store more lazily [#3121](https://github.com/NixOS/nix/issues/3121) [#15711](https://github.com/NixOS/nix/pull/15711) [#15920](https://github.com/NixOS/nix/pull/15920) + + Historically, flakes source trees have been eagerly fetched to and evaluated from the Nix store to ensure deterministic and hermetic evaluation, even if the resulting store object is not used as a derivation input. This made the implementation simpler, yet made flakes unusable in large repositories and performed unnecessary writes to the store on each change to the source tree. + + Since Nix 2.32, all I/O (excluding `path:` and `hg+:`-style inputs) for reading sources during evaluation has been funneled to their original filesystem location (or to the `~/.cache/nix/tarball-cache-v2` bare git repository for tarball-based inputs). However, the source tree was still fetched to the store -- primarily for computing the resulting content-addressed store path. In most cases, (such as importing the `nixpkgs` package set) this is not necessary. + + Touching (and hashing the NAR serialisation of) the whole source tree is unavoidable, since: + + - In case of flake inputs, `narHash` integrity must be checked eagerly. + - The `outPath` attribute of a flake must be known in advance, and for backwards compatibility must be a content-addressed store path string with [constant string context](@docroot@/language/string-context.md#string-context-constant) representing the flake source tree. + + Even within the constraints imposed by backwards compatibility requirements, there are several improvements that are achievable. To reduce the number of copies performed, Nix now hashes the input without copying first, assuming that the `.outPath` will not end up in a derivation attribute and thus would never have to be actually fetched to the store. This comes at the slight cost of doing more work in case the assumption is wrong, but results in less work in typical use cases. The evaluator continues to behave as if the copy was performed: + + - Flakes are still evaluated from the store, from the evaluator's point of view. + - `toString ./.` continues to produce a content-addressed store path string without context. + - Path resolution crossing trees located in the filesystem and in Nix's view of it (with "virtual" overlays on top) continues to work. For example, the flake source tree can contain a relative symlink pointing outside its corresponding store object (though such usage is discouraged and makes further improvements to laziness intractable). + - Reading files from the flake's `outPath` continues to work. For example, such code is well-formed and is not considered [IFD](@docroot@/language/import-from-derivation.md): + + ```nix + builtins.readFile ( /. + (builtins.unsafeDiscardStringContext self.outPath) + "/flake.nix" ) + ``` + + Similar treatment has been applied to `builtins.fetchTarball`, which no longer eagerly copies paths to the store. + `builtins.storePath` now also short-circuits on "lazy-ish" store paths and doesn't substitute unless necessary. + + This change is expected to significantly reduce disk usage required for typical evaluations and results in ~2x speedup for fetching and unpacking a nixpkgs tarball (either via `fetchTree`/flakes or via `fetchTarball`). + +- Support FreeBSD `libjail` based sandboxing, add `x86_64-freebsd` to installer [#9968](https://github.com/NixOS/nix/pull/9968) [#13281](https://github.com/NixOS/nix/pull/13281) [#15673](https://github.com/NixOS/nix/pull/15673) + + The FreeBSD build of Nix now supports build sandboxing via FreeBSD jails and is enabled by default. + A FreeBSD build has been added to the traditional installer script. The beta rust-based installer is not yet supported. + FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages and performing other tasks expected of Nix on Linux. + +## Improvements + +- HTTP/3 (QUIC) support [#15961](https://github.com/NixOS/nix/pull/15961) + + Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 (QUIC), controlled by a new [`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by default). + When enabled, Nix requests HTTP/3 and transparently falls back to HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. + The setting only takes effect when linked against a `libcurl` built with HTTP/3 support, otherwise it is ignored and Nix keeps using HTTP/2 without warning or error. + + Enable it with: + + ``` + nix.conf: http3 = true + CLI: --http3 + ``` + + Or disable with: + + ``` + nix.conf: http3 = false + CLI: --no-http3 + ``` + +- Link mimalloc for faster evaluation [#15596](https://github.com/NixOS/nix/pull/15596) + + The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. + This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. + The allocator can be disabled at build time with `-Dmimalloc=disabled`. + +- The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) + + `revCount` and `lastModified` attributes passed to the Git fetcher are no longer eagerly validated when explicitly specified. + + When not explicitly specified, `revCount` is now also a thunk value and not computed eagerly. This delays this (potentially) expensive computation until the value is actually required. + +- Configurable file-transfer retry backoff with full jitter and `Retry-After` support [#15023](https://github.com/NixOS/nix/issues/15023) [#15419](https://github.com/NixOS/nix/issues/15419) [#15449](https://github.com/NixOS/nix/pull/15449) + + File transfer retries (downloads and uploads) now use AWS-style "full jitter" exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), + and honor the `Retry-After` response header. + + Retry timing is configurable via new `nix.conf` settings: + + - [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay): base delay for transient errors + - [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited): base delay for 429/503 + - [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay): per-attempt delay ceiling + - [`filetransfer-retry-jitter`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-jitter): enable full jitter + + The existing `download-attempts` setting has been renamed to [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) to reflect that it applies to uploads as well as downloads. + The old name remains as an alias for backwards compatibility. + + Per-substituter overrides are available as store reference parameters ([`retry-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay), [`retry-delay-rate-limited`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay-rate-limited), [`retry-max-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-max-delay), [`retry-attempts`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-attempts)), e.g. `s3://my-cache?retry-attempts=8`. + +- Improve daemon socket path logic for chroot stores [#15190](https://github.com/NixOS/nix/pull/15190) + + The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). + This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. + + Example: + + ```bash + nix-daemon --store /foo/bar + ``` + + will now use a socket at: + ``` + /foo/bar/nix/var/nix/daemon-socket/socket + ``` + instead of + ``` + $NIX_STATE_DIR/daemon-socket/socket + ``` + + Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: + + - When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: + + ```bash + nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" + ``` + + - When connecting as a client, put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): + + ``` + unix://$NIX_STATE_DIR/daemon-socket/socket + ``` + +- Linux sandbox: also block `listxattr` syscalls [#15743](https://github.com/NixOS/nix/pull/15743) + + The Linux sandbox now also returns `ENOTSUP` for `listxattr`, `llistxattr` and `flistxattr`, matching the existing treatment of `getxattr`/`setxattr`/`removexattr`. + This prevents host xattrs (e.g. `security.selinux`) from leaking into builds and fixes tools such as `mkfs.ubifs` that probe xattr support via `listxattr`. + +- Support SCP-like URLs in fetchGit and type = "git" flake inputs [#14852](https://github.com/NixOS/nix/issues/14852) [#14867](https://github.com/NixOS/nix/issues/14867) [#14863](https://github.com/NixOS/nix/pull/14863) + + Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially + restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: + + ```nix + builtins.fetchGit "host:/absolute/path/to/repo" + ``` + + Nix also passes through the tilde (for home directories) verbatim: + + ```nix + builtins.fetchGit "host:~/relative/to/home" + ``` + + IPv6 addresses also supported when bracketed: + + ```nix + builtins.fetchGit "user@[::1]:~/relative/to/home" + ``` + + `builtins.fetchTree` also supports this syntax now: + + ```nix + builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } + ``` + +- `nix flake check` now supports `--print-out-paths` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) and `--out-link` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) defaulting to not creating out links if the flag is not specified. + +- Added `--skip-alive` (and `--skip-live` alias for compatibility with Lix users) option to `nix store delete` for collecting garbage within a closure [#7239](https://github.com/NixOS/nix/issues/7239) [#15236](https://github.com/NixOS/nix/pull/15236) [#15727](https://github.com/NixOS/nix/pull/15727) + + `nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. + The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. + +- `builtins.getFlake` now supports path values [#15290](https://github.com/NixOS/nix/pull/15290) + + `builtins.getFlake` now accepts path values in addition to flakerefs. This improves the usability of relative flakes, allowing you to write `builtins.getFlake ./subflake`. + This change does not allow specifying paths that are not already in the store (though they do not have valid store objects, i.e. this will not force a copy if the flake has only been hashed -- and not copied to the store). This may change in a future release. + +- `nix-profile.fish` and `nix-profile-daemon.fish` now use `$NIX_LINK` for computing the value of `NIX_PROFILE` instead of `$HOME/.nix-profile` [#14293](https://github.com/NixOS/nix/pull/14293) + +- `nix` binary now exports symbols from C bindings [#15696](https://github.com/NixOS/nix/pull/15696) + + This allows Nix plugins written against the C API to look up symbols dynamically without linking to corresponding `libnix*c.so` libraries. + +- The computed Git LFS endpoint URLs have been fixed to follow the spec [#15891](https://github.com/NixOS/nix/pull/15891) and memory usage of LFS fetches has been decreased [#15912](https://github.com/NixOS/nix/pull/15912) + +- We now verify that fetched Git LFS objects have the same OID as requested [#15845](https://github.com/NixOS/nix/pull/15845) + +- Primop documentation now includes time complexity information [#14554](https://github.com/NixOS/nix/pull/14554) + +- Improved documentation on store paths and derivation building [#14699](https://github.com/NixOS/nix/pull/14699) + +- The [build hook](@docroot@/command-ref/conf-file.md#conf-build-hook) is now killed with `SIGTERM` instead of `SIGKILL` [#15105](https://github.com/NixOS/nix/pull/15105) + +- Download/upload logs strip `userinfo` URL components [#15715](https://github.com/NixOS/nix/pull/15715) + +## Content-addressed derivations changes + + The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. + This changes the binary cache endpoints for realisations and the daemon/nix-serve protocol (gated behind a daemon protocol feature flag). + +- Realisations keyed by store path instead of hash modulo [#11897](https://github.com/NixOS/nix/issues/11897) [#12464](https://github.com/NixOS/nix/pull/12464) + + Previously, a build trace entry (realisation) was keyed by the hash modulo of the derivation. In simpler terms, derivations transitively depending on distinct fixed-output derivations with the same `outPath` would share a realisation. + + Now, build trace entries are keyed by the regular derivation store path (`.drvPath`) plus the output name. For example, instead of: + + ``` + sha256:ba7816bf8f01...!out + ``` + + The key is now: + + ``` + /nix/store/abc...-foo.drv^out + ``` + +- Removed support for "deep" realisations [#15289](https://github.com/NixOS/nix/pull/15289) + + Previously the build trace (set of "realisations") contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. + Now, it contains entries exclusively for resolved derivations. + For now, unresolved derivations will be resolved from these underlying build trace entries. + This is slower, but has the benefit of making build trace entries stateless and self-describing --- making sharing realisations easier between stores. + + This change necessitates changes to the binary cache format: + + - The directory for build traces moved from `realisations/` to `build-trace-v2/`. + - File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. + - The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. + - The deprecated `dependentRealisations` field has been removed. + + The build trace entries stored in the local SQLite database no longer have any foreign key references to store objects. + This is because the build trace entries for resolved derivations that may have been deleted need to be preserved, otherwise the outputs of other unresolved derivations will be effectively forgotten. + GC for the build trace is not yet implemented due to the lack of a clear default policy. + +- Structured signature for realisations and `path-info` [#15009](https://github.com/NixOS/nix/pull/15009) + + [Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. + `nix path-info --json --json-format 3` opts into the new version for this command. + JSON parsing accepts both the old string format and new structured format for backwards compatibility. + + This format is also used for the build trace entries in binary caches. + +- `nix realisation` command has been renamed to `nix store build-trace` [#16000](https://github.com/NixOS/nix/pull/16000) [#15948](https://github.com/NixOS/nix/pull/15948) + +## Build performance improvements + +- Make post-build-hook asynchronous [#15406](https://github.com/NixOS/nix/issues/15406) [#15451](https://github.com/NixOS/nix/pull/15451) + + The [`post-build-hook`](@docroot@/command-ref/conf-file.md#conf-post-build-hook) now runs asynchronously, without blocking the build event loop. + Dependent builds are not started until the hook finishes, but multiple hook instances are now launched concurrently -- up to the [`max-jobs`](@docroot@/command-ref/conf-file.md#conf-max-jobs) limit. + +- zstd compression now emits multi-frame output and uses less memory [#15550](https://github.com/NixOS/nix/pull/15550) + + zstd-compressed NARs are now written as a sequence of independent 16 MiB frames instead of a single large frame. + This lays the groundwork for parallel decompression in a future release without requiring caches to be repopulated, and significantly lowers peak memory use during compression + (e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + + The output remains standard zstd and is decoded unchanged by existing Nix binaries and the `zstd` CLI; compression ratio is effectively unchanged. + + Per-frame compression now uses up to 4 worker threads. For zstd this is the new default: the [`parallel-compression`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-parallel-compression) store setting defaults to `true` when `compression=zstd` (it remains `false` for other compression algorithms like `xz`). + Set `?parallel-compression=false` to opt out. + +- More parallelism for binary cache uploads [#15957](https://github.com/NixOS/nix/pull/15957) + + Uploads of NARs now start without waiting for all references to be uploaded. + Also, NARs are now uploaded in order of descending (decompressed) NAR size. + The closure invariant is still maintained by copying `.narinfo` in a topologically sorted order. + +- The derivation build scheduler memory usage reduction and performance improvements [#15611](https://github.com/NixOS/nix/pull/15611) [#15695](https://github.com/NixOS/nix/pull/15695) + + Memory usage of the derivation build scheduler has been improved to allow more state sharing. + Inefficiencies leading to quadratic complexity of scheduling build/substitution jobs have been addressed. + Scheduling resources are allocated more sparingly and freed earlier to reduce peak consumption. + + These improvements amount to ~2-8x less `nix-daemon` memory usage for typical workloads and more in larger derivation graphs, not accounting for short-lived allocations used during substitution. + + Notably, the current architecture of the build scheduler gets proportionally slower on Linux with larger heaps as derivation "builder" processes are `fork`-ed directly from the Nix process, which blocks the builder event loop for the duration of the `fork`. Thus, smaller heap of `nix-daemon` translates into faster build startups. + +- Concurrent path substitutions and eval-time fetches of the same inputs now run only once [#15555](https://github.com/NixOS/nix/pull/15555) [#15644](https://github.com/NixOS/nix/pull/15644) + + This avoids redundant work in case multiple Nix processes try to substitute/download the same resource concurrently. + +- `.narinfo` lookups in binary caches are more concurrent + + Querying the existence and path metadata in binary caches is now more asynchronous. Operations like `nix path-info` on large closures are faster and more efficient. + The build scheduler event loop now doesn't block on `.narinfo` queries, which improves performance with passthru binary caches. + +## Bug fixes + +- Fix hash collision between store paths with self-references and their zeroed-out equivalents [#15837](https://github.com/NixOS/nix/issues/15837) [#15931](https://github.com/NixOS/nix/pull/15931) + + When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. + The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. + + This release restores hashing the positions of self-references. + As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. + This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to the internal `nix::EvalState` rather than the C API wrapper struct. + This caused a segfault when the callback used the pointer with C API functions such as `nix_alloc_value()`. + The same issue affected `printValueAsJSON` and `printValueAsXML` callbacks on external values. + +- GitHub fetcher now validates URL parameters [#15304](https://github.com/NixOS/nix/issues/15304) [#15331](https://github.com/NixOS/nix/pull/15331) + + The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. + +- Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands [#15776](https://github.com/NixOS/nix/pull/15776) + + Setting [`keep-derivations`](@docroot@/command-ref/conf-file.md#conf-keep-derivations) to `true` and trying to delete a derivation with realised outputs would previously fail. + Same with [`keep-outputs`](@docroot@/command-ref/conf-file.md#conf-keep-outputs) and trying to delete an output that still has derivers. + These options no longer affect the deletion commands, and are now documented as such. + +- S3 substituters fall back to the URL's region for STS WebIdentity auth [#15594](https://github.com/NixOS/nix/pull/15594) + + When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL as a fallback for the STS endpoint region if neither `AWS_REGION` nor `AWS_DEFAULT_REGION` is set. + Previously, IRSA setups that exported `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail with a misleading "IMDS provider" error. + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a custom chain built on `aws-c-auth`. + That chain omitted two providers, breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, `AWS_ROLE_SESSION_NAME`) -- used by EKS IRSA, GitHub Actions OIDC, and any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, `AWS_CONTAINER_CREDENTIALS_FULL_URI`) -- used by ECS tasks and EKS Pod Identity. + + The typical symptom was a misleading IMDS error (`Valid credentials could not be sourced by the IMDS provider`), because IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the pre-2.33 behaviour. + As in both the old and new AWS SDK default chains, ECS and IMDS are mutually exclusive: when container credential environment variables are set, IMDS is skipped. + +- HTTP 401 and 407 responses from binary caches are no longer treated as missing files [#15877](https://github.com/NixOS/nix/pull/15877) + + Nix no longer treats `Unauthorized` and `Proxy Authentication Required` HTTP codes as an indication of a missing file. This used to be the case because AWS S3 returns 403 `Forbidden` for missing objects in unlistable buckets. 401/407 were accidentally included and this workaround is now tightly scoped to 403 responses. + +- Fixed `nixbld` gid in `/etc/group` in the Linux build sandbox when user namespaces are not supported [#15131](https://github.com/NixOS/nix/pull/15131) + +- Store garbage collection is now more robust [#15992](https://github.com/NixOS/nix/pull/15992) [#15720](https://github.com/NixOS/nix/pull/15720) [#15616](https://github.com/NixOS/nix/pull/15616) + +- Fixed deadlock for hash-mismatching fixed-output derivations [#15874](https://github.com/NixOS/nix/pull/15874) + +- `nix-copy-closure` no longer ignores `--include-outputs` flag [#15896](https://github.com/NixOS/nix/pull/15896) + +- Fixes to `recursive-nix` experimental feature + + Prior to this release, internal datastructures used to implement this feature were not used in a thread-safe manner. + Threads handling daemon connections are now reaped promptly, fixing resource leaks. + +## Contributors + +This release was made possible by the following 59 contributors: + +- Michael Wang [**(@zwang20)**](https://github.com/zwang20) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Reuben Gardos Reid [**(@ReubenJ)**](https://github.com/ReubenJ) +- StepBroBD [**(@stepbrobd)**](https://github.com/stepbrobd) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Tom [**(@thunze)**](https://github.com/thunze) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- steveoliphant [**(@steveoliphant)**](https://github.com/steveoliphant) +- espes [**(@espes)**](https://github.com/espes) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- sander [**(@sandydoo)**](https://github.com/sandydoo) +- Erik Jensen [**(@rkjnsn)**](https://github.com/rkjnsn) +- Cameron Will [**(@cwill747)**](https://github.com/cwill747) +- Maciej Krüger [**(@mkg20001)**](https://github.com/mkg20001) +- Dror Speiser [**(@drorspei)**](https://github.com/drorspei) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- Audrey Dutcher [**(@rhelmot)**](https://github.com/rhelmot) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- TyIsI [**(@TyIsI)**](https://github.com/TyIsI) +- Adam Kliś [**(@BonusPlay)**](https://github.com/BonusPlay) +- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- ryota2357 [**(@ryota2357)**](https://github.com/ryota2357) +- LIN, Jian [**(@jian-lin)**](https://github.com/jian-lin) +- znmz [**(@znmz)**](https://github.com/znmz) +- Felix Stupp [**(@Zocker1999NET)**](https://github.com/Zocker1999NET) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- Antonio Nuno Monteiro [**(@anmonteiro)**](https://github.com/anmonteiro) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- adisbladis [**(@adisbladis)**](https://github.com/adisbladis) +- Luna Nova [**(@LunNova)**](https://github.com/LunNova) +- Riccardo Mazzarini [**(@noib3)**](https://github.com/noib3) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Dario [**(@dve00)**](https://github.com/dve00) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Paul Sbarra [**(@tones111)**](https://github.com/tones111) +- edef [**(@edef1c)**](https://github.com/edef1c) +- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Alex Ionescu [**(@aionescu)**](https://github.com/aionescu) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Pierre Penninckx [**(@ibizaman)**](https://github.com/ibizaman) +- Leonard Sheng Sheng Lee [**(@sheeeng)**](https://github.com/sheeeng) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Lennart Kolmodin [**(@kolmodin)**](https://github.com/kolmodin) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Peter Collingbourne [**(@pcc)**](https://github.com/pcc) +- Simon Žlender [**(@szlend)**](https://github.com/szlend) +- Lily Foster [**(@lilyinstarlight)**](https://github.com/lilyinstarlight) +- randomizedcoder [**(@randomizedcoder)**](https://github.com/randomizedcoder) +- Krish Jaiswal From eb1602c8610e92315532acdbaf6ff890d785ba47 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 6 Jul 2026 00:44:35 +0300 Subject: [PATCH 309/364] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index aa5388f63762..3a05135cd86d 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.0 +2.36.0 From 9611c1cfdb4cfe89291f2378441763c7320b2c44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:12:11 +0000 Subject: [PATCH 310/364] build(deps): bump korthout/backport-action from 4.5.2 to 4.6.0 Bumps [korthout/backport-action](https://github.com/korthout/backport-action) from 4.5.2 to 4.6.0. - [Release notes](https://github.com/korthout/backport-action/releases) - [Commits](https://github.com/korthout/backport-action/compare/66065406958f46e82238fd59546f5a99e69e22aa...2e830a1d0b8269505846ddd407a70876913ad1f8) --- updated-dependencies: - dependency-name: korthout/backport-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index a4801d756095..9bc2ee17e672 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -26,7 +26,7 @@ jobs: # required to find all branches fetch-depth: 0 - name: Create backport PRs - uses: korthout/backport-action@66065406958f46e82238fd59546f5a99e69e22aa # v4.5.2 + uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8 # v4.6.0 id: backport with: # Config README: https://github.com/korthout/backport-action#backport-action From bb10f6aabaaad457b558a41ab99648462a359c35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:12:17 +0000 Subject: [PATCH 311/364] build(deps): bump docker/login-action from 4.0.0 to 4.4.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.0.0 to 4.4.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/b45d80f862d83dbcd57f89517bcf500b2ab88fb2...af1e73f918a031802d376d3c8bbc3fe56130a9b0) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/upload-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index 68af33081089..332115cfd992 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -51,12 +51,12 @@ jobs: echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null sudo systemctl restart docker - name: Login to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} From cce7c670152dec9d6766ed715bd22b8b062ace1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:13:25 +0000 Subject: [PATCH 312/364] build(deps): bump the flake-inputs group with 3 updates Bumps the flake-inputs group with 3 updates: [flake-parts](https://github.com/hercules-ci/flake-parts), [git-hooks-nix](https://github.com/cachix/git-hooks.nix) and [nixpkgs](https://github.com/NixOS/nixpkgs). Updates `flake-parts` from `f7c1a2d` to `17c9d6c` - [Commits](https://github.com/hercules-ci/flake-parts/compare/f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb...17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e) Updates `git-hooks-nix` from `3bbec39` to `bca82ca` - [Commits](https://github.com/cachix/git-hooks.nix/compare/3bbec39bc90eadfa031e6f3b77272f3f60803e39...bca82caa46d5ec0f5d422c61fb1e30bc51313cbe) Updates `nixpkgs` from `714a5f8` to `a50de1b` - [Commits](https://github.com/NixOS/nixpkgs/commits) --- updated-dependencies: - dependency-name: flake-parts dependency-version: 17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e dependency-type: direct:production dependency-group: flake-inputs - dependency-name: git-hooks-nix dependency-version: bca82caa46d5ec0f5d422c61fb1e30bc51313cbe dependency-type: direct:production dependency-group: flake-inputs - dependency-name: nixpkgs dependency-version: a50de1b7d8a586adc18d2395c19de7d6058e6030 dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] --- flake.lock | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/flake.lock b/flake.lock index 1f946b3532db..7dcbdd60af35 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1778716662, - "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=", + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", "type": "github" }, "original": { @@ -39,17 +39,16 @@ "git-hooks-nix": { "inputs": { "flake-compat": [], - "gitignore": [], "nixpkgs": [ "nixpkgs" ] }, "locked": { - "lastModified": 1781733627, - "narHash": "sha256-U3yTuGBnmXvXoQI3qkpfEDsn9RovQPAjN7ndRco+3u0=", + "lastModified": 1783008725, + "narHash": "sha256-jGiy6+sxjNWXSjp25uoJuNfyH9zBK1PEDY0lVoL4ibQ=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "3bbec39bc90eadfa031e6f3b77272f3f60803e39", + "rev": "bca82caa46d5ec0f5d422c61fb1e30bc51313cbe", "type": "github" }, "original": { @@ -60,11 +59,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1782535326, - "narHash": "sha256-r4TA57SL7nvj1R+GY/FCLwFU48w9IixTQlzBTIYkt8E=", - "rev": "714a5f8c4ead6b31148d829288440ed033ccc041", + "lastModified": 1783148766, + "narHash": "sha256-H9+N+GFtsbVC8ZniHliChM7ndizxtqVZs6bnGOLM3WQ=", + "rev": "a50de1b7d8a586adc18d2395c19de7d6058e6030", "type": "tarball", - "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.3494.714a5f8c4ead/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.4193.a50de1b7d8a5/nixexprs.tar.xz" }, "original": { "type": "tarball", From 202510df6a91bcda610eae8ea6415c4d10701e5c Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Fri, 5 Jun 2026 15:16:10 -0400 Subject: [PATCH 313/364] Establish Automation/AI policy This is adapted from the [nixpkgs policy](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md#automationai-policy), but with additional restrictions around authorship of human communication. Resolves #15340 Signed-off-by: Lisanna Dettwyler --- .github/ISSUE_TEMPLATE/bug_report.md | 2 + .github/ISSUE_TEMPLATE/feature_request.md | 2 + .github/ISSUE_TEMPLATE/installer.md | 2 + .../ISSUE_TEMPLATE/missing_documentation.md | 2 + .github/PULL_REQUEST_TEMPLATE.md | 1 + CONTRIBUTING.md | 61 +++++++++++++++++++ 6 files changed, 70 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index af94c3e9e5bb..d802a82c2ecd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -44,10 +44,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open bug issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index fe9f9dd209d4..2238b88386ae 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -29,10 +29,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open feature issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index 070e0bd9b25b..965d4db6fd96 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -37,10 +37,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open installer issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 4e05b626d398..5675bf2391a6 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -21,10 +21,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open documentation issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c155bf8bfa4f..e861dcfc5fa0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,7 @@ so you understand the process and the expectations. - what information to include in commit messages - proper attribution - volunteering contributions effectively +- AI/automation policy - how to get help and our review process. PR stuck in review? We have two Nix team meetings per week online that are open for everyone in a jitsi conference: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c170ae4a770..f6679e0a8c93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. + * Review the **Automation/AI Policy** below. 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. @@ -87,6 +88,66 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). +## Automation/AI policy + +Every contribution to Nix and related development venues, including code, documentation, and communication on GitHub and Matrix, must have a responsible person in the loop who is accountable for that contribution and reviews it before submission, and must transparently disclose any non‐trivial use of automation to produce it, including but not limited to LLM‐based AI tools. + +Human communication must remain human. Pull request / issue descriptions and comments, documentation, commit messages, and code comments must be human-authored. + +The following sections give more detail. + +### Scope + +Any use of automated tools to generate non‐trivial amounts of output as part of a contribution, in whole or in part, verbatim or edited, is covered by this policy, except as listed in the Exemptions section. +Both LLM‐based AI tools and hand‐written automation are covered. +Contributions include code and documentation in commits, commit messages, pull request summaries and reviews, issue and vulnerability reports, GitHub comments, Matrix messages, and Discourse posts. +The covered venues are the GitHub repositories for Nix and related projects under the jurisdiction of the Nix team, Matrix rooms that are focused on development of those projects, and Discourse topics about Nix development. + +PRs that seek to address issues that appear easier to fix, such as those marked with [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue), are held to the same standard as other issues. +Just because the problem seems "easy" and more likely to be successfully fixed by an unsupervised agent does not mean bending the rules is permissible. +Even the most trivial changes still must have a responsible person in the loop. + +### Accountability + +Everyone who submits a contribution to Nix is responsible for it, regardless of the use of automated tooling. +Before submission, they must establish a reasonable level of understanding of the contribution and expectation of its correctness. +A contributor submitting a contribution intended for inclusion in Nix is also responsible for ensuring that it is [appropriately licensed](https://github.com/NixOS/nix/blob/master/COPYING) and credited, and not encumbered by any incompatible copyright. + +When output from automated tooling is used in contributions, a contributor must establish confidence in that output. + +This policy applies equally to any further discussion of a contribution. +Comments and reviews must separately satisfy the same requirements of understanding, review, and disclosure. +Contributors are expected to be able to answer questions about their contribution and respond to feedback appropriately, **without simply forwarding messages back and forth to automated tools**. + +It is not permitted to submit automated contributions without any manual review or intervention, outside of standard community automation. +Automation without any manual review must not be used as the sole arbiter of whether to merge a change. + +### Transparency + +All covered use of automated tooling for a contribution must be disclosed as part of that contribution. + +In the case of LLM‐based AI tooling used for commits, this **must** be in the form of an `Assisted-by:` Git commit trailer, including at least the tool name and the primary model name and version used for the contribution. When using unreleased models, it is acceptable to say "unspecified". +A `Co-authored-by:` trailer does not satisfy this policy. + +Any adequate form of disclosure is permitted for other kinds of tooling and contribution. + +### Exemptions + +The following situations are fully or partially exempt: + +* Use of standard deterministic editor/IDE/formatter/text transformation tooling to produce changes that the author manually reviews and understands is exempt, including inline "auto‐completion" (even if LLM‐based) of short, rote snippets of text that do not contribute anything beyond boilerplate the author would have written anyway, and spelling and grammar checkers. + +* Use of standard community automation is exempt (e.g. dependabot). + +* Use of AI tools for research, testing, debugging, or review is out of scope, if no substantial amount of their output is included in the resulting contribution. + However, if these tools had a significant technical influence on your contribution, you are still responsible for it per the Accountability section, and are expected to disclose this where relevant. + +* Use of machine translation for commit and pull request descriptions and comments is exempt from the requirement to understand the translated output. + However, the requirements of appropriate confidence in the original text, responsibility, and disclosure still apply, and you should additionally include the original untranslated contribution. + +* Use of automation in a contribution clearly marked as not being ready for merge (e.g. a draft pull request) is exempt from the requirement for full self‐review, as long as some amount of review has been done and it is expected that the requirements will be met by the time it is marked as ready. + This does not waive any other requirement. + ## Making changes to the Nix manual The Nix reference manual is hosted on https://nix.dev/manual/nix. From ef7e222511bd01b3628b9526ca2c95dd78927c0d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 6 Jul 2026 22:03:11 +0300 Subject: [PATCH 314/364] Make emptyBindings const and constinit, don't modify emptyBindings We have a bug with it being modified, which really should not happen. By making it const constinit such bugs would become a segfault. Co-authored-by: Eelco Dolstra --- src/libcmd/common-eval-args.cc | 2 +- src/libcmd/include/nix/cmd/common-eval-args.hh | 2 +- src/libcmd/include/nix/cmd/repl.hh | 2 +- src/libcmd/installable-attr-path.cc | 2 +- src/libexpr/attr-path.cc | 2 +- src/libexpr/attr-set.cc | 5 +++-- src/libexpr/eval.cc | 6 +++++- src/libexpr/get-drvs.cc | 4 ++-- src/libexpr/include/nix/expr/attr-path.hh | 2 +- src/libexpr/include/nix/expr/attr-set.hh | 16 ++++++++++------ src/libexpr/include/nix/expr/get-drvs.hh | 2 +- src/libexpr/include/nix/expr/value.hh | 10 +++++----- src/libutil/include/nix/util/pos-idx.hh | 4 ++-- src/nix/flake.cc | 2 +- src/nix/nix-env/nix-env.cc | 4 ++-- src/nix/nix-env/user-env.cc | 2 +- src/nix/nix-instantiate/nix-instantiate.cc | 4 ++-- src/nix/prefetch.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 19 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 984bed34882e..f60c49d5e682 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -150,7 +150,7 @@ MixEvalArgs::MixEvalArgs() }); } -Bindings * MixEvalArgs::getAutoArgs(EvalState & state) +const Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); for (auto & [name, arg] : autoArgs) { diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 14897158ae6d..c265c834fb61 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -52,7 +52,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair MixEvalArgs(); - Bindings * getAutoArgs(EvalState & state); + const Bindings * getAutoArgs(EvalState & state); LookupPath lookupPath; diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index 81c7b8df5a2d..5966a56a9574 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -9,7 +9,7 @@ namespace nix { struct AbstractNixRepl { ref state; - Bindings * autoArgs; + const Bindings * autoArgs; AbstractNixRepl(ref state) : state(state) diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index aa5da2e56646..d2d696c5cb8d 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -43,7 +43,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() return {*derivedPathWithInfo}; } - Bindings & autoArgs = *cmd.getAutoArgs(*state); + const Bindings & autoArgs = *cmd.getAutoArgs(*state); PackageInfos packageInfos; getDerivations(*state, *v, "", autoArgs, packageInfos, false); diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 8c2fcd327e77..d512baa92493 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -55,7 +55,7 @@ std::vector AttrPath::resolve(EvalState & state) const } std::pair -findAlongAttrPath(EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn) +findAlongAttrPath(EvalState & state, const std::string & attrPath, const Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 92b67f6ad25e..08412c7be9ca 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -5,7 +5,7 @@ namespace nix { -Bindings Bindings::emptyBindings; +const constinit Bindings Bindings::emptyBindings; /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings @@ -13,7 +13,8 @@ Bindings Bindings::emptyBindings; Bindings * EvalMemory::allocBindings(size_t capacity) { if (capacity == 0) - return &Bindings::emptyBindings; + /* Swear that we are not going to modify this. */ + return const_cast(&Bindings::emptyBindings); if (capacity > std::numeric_limits::max()) throw Error("attribute set of size %d is too big", capacity); stats.nrAttrsets++; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 29ecec69ee64..4bcb409707f7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1359,7 +1359,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) sort = true; } - bindings.bindings->pos = pos; + /* FIXME: Currently we can't track the positions of empty bindings. A way + to fix this is to store the position in the Value storage with more + clever bitpacking (we have spare 32 bits in the Bindings * variant). */ + if (bindings.bindings != &Bindings::emptyBindings) + bindings.bindings->pos = pos; v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted()); } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1acc591b05b7..6416713f7b3d 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -393,7 +393,7 @@ static void getDerivations( EvalState & state, Value & vIn, const std::string & pathPrefix, - Bindings & autoArgs, + const Bindings & autoArgs, PackageInfos & drvs, Done & done, bool ignoreAssertionFailures) @@ -464,7 +464,7 @@ void getDerivations( EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, + const Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures) { diff --git a/src/libexpr/include/nix/expr/attr-path.hh b/src/libexpr/include/nix/expr/attr-path.hh index fd48705b8b7b..ab765ec1b1a2 100644 --- a/src/libexpr/include/nix/expr/attr-path.hh +++ b/src/libexpr/include/nix/expr/attr-path.hh @@ -12,7 +12,7 @@ MakeError(AttrPathNotFound, Error); MakeError(NoPositionInfo, Error); std::pair -findAlongAttrPath(EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn); +findAlongAttrPath(EvalState & state, const std::string & attrPath, const Bindings & autoArgs, Value & vIn); /** * Heuristic to find the filename and lineno or a nix value. diff --git a/src/libexpr/include/nix/expr/attr-set.hh b/src/libexpr/include/nix/expr/attr-set.hh index 4d3821feda97..38d654282a79 100644 --- a/src/libexpr/include/nix/expr/attr-set.hh +++ b/src/libexpr/include/nix/expr/attr-set.hh @@ -29,11 +29,15 @@ struct Attr Symbol name; PosIdx pos; Value * value = nullptr; + Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name) , pos(pos) - , value(value) {}; - Attr() {}; + , value(value) + { + } + + constexpr Attr() {} auto operator<=>(const Attr & a) const { @@ -70,7 +74,7 @@ public: * An instance of bindings objects with 0 attributes. * This object must never be modified. */ - static Bindings emptyBindings; + static const constinit Bindings emptyBindings; private: /** @@ -101,7 +105,7 @@ private: */ Attr attrs[0]; - Bindings() = default; + constexpr Bindings() = default; Bindings(const Bindings &) = delete; Bindings(Bindings &&) = delete; Bindings & operator=(const Bindings &) = delete; @@ -550,14 +554,14 @@ public: Value & alloc(std::string_view name, PosIdx pos = noPos); - Bindings * finish() + const Bindings * finish() { bindings->sort(); finishSizeIfNecessary(); return bindings; } - Bindings * alreadySorted() + const Bindings * alreadySorted() { finishSizeIfNecessary(); return bindings; diff --git a/src/libexpr/include/nix/expr/get-drvs.hh b/src/libexpr/include/nix/expr/get-drvs.hh index 4beccabe2ad3..8e25c25e1aa7 100644 --- a/src/libexpr/include/nix/expr/get-drvs.hh +++ b/src/libexpr/include/nix/expr/get-drvs.hh @@ -112,7 +112,7 @@ void getDerivations( EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, + const Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures); diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 913a44bb418b..e1c7725624f5 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -505,7 +505,7 @@ struct PayloadTypeToInternalType MACRO(ValueBase::StringWithContext, string, tString) \ MACRO(ValueBase::Path, path, tPath) \ MACRO(ValueBase::Null, null_, tNull) \ - MACRO(Bindings *, attrs, tAttrs) \ + MACRO(const Bindings *, attrs, tAttrs) \ MACRO(ValueBase::List, bigList, tListN) \ MACRO(ValueBase::SmallList, smallList, tListSmall) \ MACRO(ValueBase::ClosureThunk, thunk, tThunk) \ @@ -900,7 +900,7 @@ protected: primOp = std::bit_cast(payload[1]); } - void getStorage(Bindings *& attrs) const noexcept + void getStorage(const Bindings *& attrs) const noexcept { Payload payload = loadPayload(); attrs = std::bit_cast(payload[1]); @@ -963,7 +963,7 @@ protected: setSingleDWordPayload(std::bit_cast(primOp)); } - void setStorage(Bindings * bindings) noexcept + void setStorage(const Bindings * bindings) noexcept { setSingleDWordPayload(std::bit_cast(bindings)); } @@ -1361,7 +1361,7 @@ public: setStorage(Null{}); } - inline void mkAttrs(Bindings * a) noexcept + inline void mkAttrs(const Bindings * a) noexcept { setStorage(a); } @@ -1487,7 +1487,7 @@ public: const Bindings * attrs() const noexcept { - return getStorage(); + return getStorage(); } const PrimOp * primOp() const noexcept diff --git a/src/libutil/include/nix/util/pos-idx.hh b/src/libutil/include/nix/util/pos-idx.hh index 8e668176c619..299f703d98af 100644 --- a/src/libutil/include/nix/util/pos-idx.hh +++ b/src/libutil/include/nix/util/pos-idx.hh @@ -15,13 +15,13 @@ class PosIdx private: uint32_t id; - explicit PosIdx(uint32_t id) + constexpr explicit PosIdx(uint32_t id) : id(id) { } public: - PosIdx() + constexpr PosIdx() : id(0) { } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e56c3d86a8c3..3fae1f0d6a30 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -530,7 +530,7 @@ struct CmdFlakeCheck : FlakeCommand, MixPrintOutPaths, MixOutLinkBase auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS configuration '%s'", attrPath)); - Bindings & bindings = Bindings::emptyBindings; + const Bindings & bindings = Bindings::emptyBindings; auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; state->forceValue(*vToplevel, pos); if (!state->isDerivation(*vToplevel)) diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index abe04d96341c..6663cb2db67a 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -76,7 +76,7 @@ struct InstallSourceInfo std::shared_ptr nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ std::filesystem::path profile; /* for srcProfile */ std::string systemFilter; /* for srcNixExprDrvs */ - Bindings * autoArgs; + const Bindings * autoArgs; }; struct Globals @@ -207,7 +207,7 @@ static void loadDerivations( EvalState & state, const SourcePath & nixExprPath, std::string systemFilter, - Bindings & autoArgs, + const Bindings & autoArgs, const std::string & pathPrefix, PackageInfos & elems) { diff --git a/src/nix/nix-env/user-env.cc b/src/nix/nix-env/user-env.cc index cb74a36d0d4c..9d0cdee48fe5 100644 --- a/src/nix/nix-env/user-env.cc +++ b/src/nix/nix-env/user-env.cc @@ -23,7 +23,7 @@ PackageInfos queryInstalled(EvalState & state, const std::filesystem::path & use if (pathExists(manifestFile)) { Value v; state.evalFile(state.rootPath(CanonPath(manifestFile.string())).resolveSymlinks(), v); - Bindings & bindings = Bindings::emptyBindings; + const Bindings & bindings = Bindings::emptyBindings; getDerivations(state, v, "", bindings, elems, false); } return elems; diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index dee79bcbfc2c..b180472c1275 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -27,7 +27,7 @@ void processExpr( const Strings & attrPaths, bool parseOnly, bool strict, - Bindings & autoArgs, + const Bindings & autoArgs, bool evalOnly, OutputKind output, bool location, @@ -172,7 +172,7 @@ static int main_nix_instantiate(int argc, char ** argv) auto state = std::make_shared(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; - Bindings & autoArgs = *myArgs.getAutoArgs(*state); + const Bindings & autoArgs = *myArgs.getAutoArgs(*state); if (attrPaths.empty()) attrPaths = {""}; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 2cd4377dd3c0..bceb8c3bb3db 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -205,7 +205,7 @@ static int main_nix_prefetch_url(int argc, char ** argv) auto store = openStore(); auto state = std::make_shared(myArgs.lookupPath, store, fetchSettings, evalSettings); - Bindings & autoArgs = *myArgs.getAutoArgs(*state); + const Bindings & autoArgs = *myArgs.getAutoArgs(*state); /* If -A is given, get the URL from the specified Nix expression. */ diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3136f4d5fd15..1b2873d7f9c0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -201,7 +201,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto state = std::make_shared(LookupPath{}, store, fetchSettings, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); - Bindings & bindings = Bindings::emptyBindings; + const Bindings & bindings = Bindings::emptyBindings; auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; return store->parseStorePath( From a1238bd3968cc4771b04e82b9a4bc0f4dae50a10 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 7 Jul 2026 21:17:42 +0300 Subject: [PATCH 315/364] libfetchers/git-utils: Avoid intermediate git_buf and git_packbuilder_write_buf We can instead do a git_packbuilder_foreach (what git_packbuilder_write_buf uses internally already) to write things to the indexer in chunks without having to allocate them anywhere. Also add an early bail-out in flush() in case the mempack backend has no objects to write. This was surfaced by a new debug print which includes a bit more information about the name of the packfile being written and the number of objects/deltas. --- src/libfetchers/git-utils.cc | 105 ++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 26 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 286d31130e36..1e2dc25281c1 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -99,6 +99,32 @@ struct GitError final : public CloneableError } }; +struct GitIndexerSink final : public BufferedSink +{ + git_indexer * indexer; + git_indexer_progress stats{}; + + GitIndexerSink(git_indexer * indexer) + : BufferedSink(1 * 1024 * 1024) + , indexer(indexer) + { + assert(indexer); + } + + GitIndexerSink(GitIndexerSink &&) = delete; + GitIndexerSink(const GitIndexerSink &) = delete; + GitIndexerSink & operator=(GitIndexerSink &&) = delete; + GitIndexerSink & operator=(const GitIndexerSink &) = delete; + ~GitIndexerSink() = default; + + void writeUnbuffered(std::string_view data) override + { + checkInterrupt(); + if (git_indexer_append(indexer, data.data(), data.size(), &stats)) + throw GitError("appending to git packfile index"); + } +}; + } // namespace typedef std::unique_ptr> Repository; @@ -337,46 +363,67 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this void flush() override { + std::size_t objectCount; + if (git_mempack_object_count(&objectCount, mempackBackend)) + throw GitError("querying the number of objects in a git memory packer backend"); + + if (!objectCount) + /* Nothing to do. */ + return; + checkInterrupt(); - git_buf buf = GIT_BUF_INIT; - Finally _disposeBuf{[&] { git_buf_dispose(&buf); }}; PackBuilder packBuilder; PackBuilderContext packBuilderContext; - git_packbuilder_new(Setter(packBuilder), *this); - git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext); + if (git_packbuilder_new(Setter(packBuilder), *this)) + throw GitError("creating git pack builder"); + + if (git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext)) + throw GitError("setting git pack builder callbacks"); + git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */); packBuilderContext.handleException( "preparing packfile", git_mempack_write_thin_pack(mempackBackend, packBuilder.get())); checkInterrupt(); - packBuilderContext.handleException("writing packfile", git_packbuilder_write_buf(&buf, packBuilder.get())); - checkInterrupt(); - std::string repo_path = std::string(git_repository_path(repo.get())); - while (!repo_path.empty() && repo_path.back() == '/') - repo_path.pop_back(); - std::string pack_dir_path = repo_path + "/objects/pack"; + auto packFilesPath = std::filesystem::path(git_repository_path(repo.get())) / "objects/pack"; - // TODO (performance): could the indexing be done in a separate thread? - // we'd need a more streaming variation of - // git_packbuilder_write_buf, or incur the cost of - // copying parts of the buffer to a separate thread. - // (synchronously on the git_packbuilder_write_buf thread) Indexer indexer; - git_indexer_progress stats; - if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) + if (git_indexer_new(Setter(indexer), packFilesPath.c_str(), 0, nullptr, nullptr)) throw GitError("creating git packfile indexer"); - // TODO: provide index callback for checkInterrupt() termination - // though this is about an order of magnitude faster than the packbuilder - // expect up to 1 sec latency due to uninterruptible git_indexer_append. - constexpr size_t chunkSize = 128 * 1024; - for (size_t offset = 0; offset < buf.size; offset += chunkSize) { - if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats)) - throw GitError("appending to git packfile index"); - checkInterrupt(); - } + struct State + { + Indexer & indexer; + PackBuilderContext & packBuilderContext; + GitIndexerSink sink{indexer.get()}; + }; + + State state{ + .indexer = indexer, + .packBuilderContext = packBuilderContext, + }; + + packBuilderContext.handleException( + "writing packfile", + git_packbuilder_foreach( + packBuilder.get(), + [](void * buf, size_t size, void * payload) -> int { + auto & state = *static_cast(payload); + try { + state.sink(std::string_view(static_cast(buf), size)); + } catch (...) { + state.packBuilderContext.exception = std::current_exception(); + return GIT_EUSER; + } + return GIT_OK; + }, + &state)); + + state.sink.flush(); + + auto & stats = state.sink.stats; if (git_indexer_commit(indexer.get(), &stats)) throw GitError("committing git packfile index"); @@ -384,6 +431,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_mempack_reset(mempackBackend)) throw GitError("resetting git mempack backend"); + debug( + "committed index and pack file to pack-%s.{idx,pack}, objects = %d, deltas = %d", + git_indexer_name(indexer.get()), + stats.total_objects, + stats.total_deltas); + checkInterrupt(); } From b8bc06310c4ad8480c13b3bbbb02e4422fbd4a69 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 7 Jul 2026 21:17:52 +0300 Subject: [PATCH 316/364] libfetchers: Make GitFileSystemObjectSinkImpl non-copyable and non-movable It's self-referential (this pointer is passed off to the thread pool). --- src/libfetchers/git-utils.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 1e2dc25281c1..b42feaaba58f 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1204,6 +1204,11 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink { } + GitFileSystemObjectSinkImpl(GitFileSystemObjectSinkImpl &&) = delete; + GitFileSystemObjectSinkImpl(const GitFileSystemObjectSinkImpl &) = delete; + GitFileSystemObjectSinkImpl & operator=(GitFileSystemObjectSinkImpl &&) = delete; + GitFileSystemObjectSinkImpl & operator=(const GitFileSystemObjectSinkImpl &) = delete; + ~GitFileSystemObjectSinkImpl() { // Make sure the worker threads are destroyed before any state From 6e925cfa6eb83e82b14e0c6802648c7045a14363 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 7 Jul 2026 21:17:59 +0300 Subject: [PATCH 317/364] libfetchers: Simplify tarball unpacking to tarball cache, better align file overwriting semantics with libarchive All the indexing operations now run on the main thread, which enqueues the writes - so we get the deterministic behavior for free. This allows us to get rid of a lot of code duplication without giving up on performance (writing out the tree structure in memory isn't the bottleneck - blob hashing is). Also handle symlinks synchronously and with the same repo, this way we include the tiny symlinks in the same packfile as the trees (now also written *this repo). --- src/libfetchers/git-utils.cc | 155 ++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 76 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index b42feaaba58f..50df7861bcf7 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -45,6 +45,7 @@ #include #include #include +#include namespace std { @@ -1246,27 +1247,30 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink } }; - size_t nextId = 0; // for Child.id + /* FIXME: Most of this logic is independent from git. Come up with a tree sink interface + and an adapter for a ExtendedFileSystemObjectSink that implements the tarball unpacking + semantics (i.e. overwriting of entries). Also deduplicate with MemorySourceAccessor. */ struct Child { git_filemode_t mode; - std::variant file; + std::variant> file; - /// Sequential numbering of the file in the tarball. This is - /// used to make sure we only import the latest version of a - /// path. - size_t id{0}; - }; - - struct State - { - Directory root; + const git_oid & getOid() const & + { + return std::visit( + overloaded{ + [](const Directory & dir) -> const git_oid & { return dir.oid.value(); }, + [](const git_oid & oid) -> const git_oid & { return oid; }, + [](const std::shared_future & oid) -> const git_oid & { return oid.get(); }, + }, + file); + } }; - Sync _state; + Directory root; - void addNode(State & state, const CanonPath & path, Child && child) + void addNode(const CanonPath & path, Child && child) { if (path.isRoot()) throw Error("cannot create a file at the root of the git repository"); @@ -1274,7 +1278,7 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink auto parent = path.parent(); assert(parent); - Directory * cur = &state.root; + Directory * cur = &root; for (auto & i : *parent) { auto child = std::get_if( @@ -1285,31 +1289,29 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink } std::string name(*path.baseName()); + auto prev = cur->children.find(name); - if (auto prev = cur->children.find(name); prev == cur->children.end() || prev->second.id < child.id) - cur->children.insert_or_assign(name, std::move(child)); - } + if (prev == cur->children.end()) { + cur->children.insert_or_assign(std::move(name), std::move(child)); + return; + } - /* Set the object ID of a reserved leaf, skipping if it was superseded (id changed) meanwhile. */ - void setNodeOid(State & state, const CanonPath & path, const git_oid & oid, size_t id) - { - auto parent = path.parent(); - assert(parent); + /* Overwriting part of the tree. We'd like to behave somewhat + similarly to libarchive without ARCHIVE_EXTRACT_NO_OVERWRITE. */ + const auto & prevChild = prev->second; - Directory * cur = &state.root; - for (auto & name : *parent) { - auto i = cur->children.find(name); - if (i == cur->children.end()) + /* libarchive tries to unlink an entry, which only succeeds on empty + trees - so behave the same way. Everything else is fair game. */ + if (const auto * maybePrevDir = std::get_if(&prevChild.file)) { + /* "Replacing" directory with a directory is always a-ok. */ + if (std::holds_alternative(child.file)) return; - auto dir = std::get_if(&i->second.file); - if (!dir) - return; - cur = dir; + + if (!maybePrevDir->children.empty()) + throw Error("cannot create '%1%', conflicting non-empty directory", path.rel()); } - auto i = cur->children.find(*path.baseName()); - if (i != cur->children.end() && i->second.id == id) - i->second.file = oid; + cur->children.insert_or_assign(std::move(name), std::move(child)); } void createRegularFile(const CanonPath & path, fun func) override @@ -1380,12 +1382,6 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink func(*crf); - auto id = nextId++; - auto mode = crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB; - - /* Reserve the node now, in order; workers fill the oid later. */ - addNode(*_state.lock(), crf->path, Child{mode, git_oid{}, id}); - if (crf->stream) { /* Finish the slow path by creating the blob object synchronously. Call .release(), since git_blob_create_from_stream_commit @@ -1393,43 +1389,52 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink git_oid oid; if (git_blob_create_from_stream_commit(&oid, crf->stream.release())) throw GitError("creating a blob object for '%s'", path); - setNodeOid(*_state.lock(), crf->path, oid, id); + addNode(crf->path, Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid}); return; } - /* Fast path: create the blob object in a separate thread. */ - workers.enqueue([this, crf{std::move(crf)}, id]() { - auto repo(repoPool.get()); - - git_oid oid; - if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size())) - throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path); - - setNodeOid(*_state.lock(), crf->path, oid, id); - }); + std::promise promise; + addNode( + crf->path, + Child{ + crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, + promise.get_future(), + }); + + /* Fast path: create the blob object in a separate thread. + FIXME: Ugly, make ThreadPool use std::move_only_function. */ + workers.enqueue( + [this, crf{std::move(crf)}, promise = make_ref(std::move(promise))]() mutable { + auto repo(repoPool.get()); + + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size())) + throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path); + + /* We don't generally bother with exceptions because those will + be propagated by the thread pool during .process(). */ + promise->set_value(oid); + }); } void createDirectory(const CanonPath & path) override { if (path.isRoot()) return; - auto state(_state.lock()); - addNode(*state, path, {GIT_FILEMODE_TREE, Directory()}); + addNode(path, {GIT_FILEMODE_TREE, Directory()}); } void createSymlink(const CanonPath & path, const std::string & target) override { - auto id = nextId++; - addNode(*_state.lock(), path, Child{GIT_FILEMODE_LINK, git_oid{}, id}); - workers.enqueue([this, path, target, id]() { - auto repo(repoPool.get()); - - git_oid oid; - if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) - throw GitError("creating a blob object for tarball symlink member '%s'", path); - - setNodeOid(*_state.lock(), path, oid, id); - }); + /* Symlinks are written to the this repo instance, the mempack backend + for which includes the trees. This way we flush both symlinks and + trees to the same packfile. Doing this synchronously isn't expensive + because symlinks are tiny, so hashing them is cheap. */ + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, requireCString(target), target.size())) + throw GitError("creating a blob object for tarball symlink member '%s'", path); + + addNode(path, Child{GIT_FILEMODE_LINK, oid}); } std::map hardLinks; @@ -1445,16 +1450,14 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink /* Create hard links. */ { - auto state(_state.lock()); for (auto & [path, target] : hardLinks) { if (target.isRoot()) continue; try { - auto child = state->root.lookup(target); - auto oid = std::get_if(&child.file); - if (!oid) + const auto & child = root.lookup(target); + if (std::holds_alternative(child.file)) throw Error("cannot create a hard link to a directory"); - addNode(*state, path, {child.mode, *oid}); + addNode(path, {child.mode, child.getOid()}); } catch (Error & e) { e.addTrace(nullptr, "while creating a hard link from '%s' to '%s'", path, target); throw; @@ -1468,30 +1471,30 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink ThreadPool workers{repos.size()}; for (auto & repo : repos) workers.enqueue([repo]() { repo->flush(); }); + workers.enqueue([repo = repo]() { repo->flush(); }); workers.process(); } // Write the Git trees to disk. Would be nice to have this multithreaded too, but that's hard because a tree // can't refer to an object that hasn't been written yet. Also it doesn't make a big difference for performance. - auto repo(repoPool.get()); - [&](this const auto & visit, Directory & node) -> void { + [&, &repo = *repo](this const auto & visit, Directory & node) -> void { checkInterrupt(); // Write the child directories. for (auto & child : node.children) if (auto dir = std::get_if(&child.second.file)) + /* TODO: Limit recursion depth? */ visit(*dir); // Write this directory. git_treebuilder * b; - if (git_treebuilder_new(&b, *repo, nullptr)) + if (git_treebuilder_new(&b, repo, nullptr)) throw GitError("creating a tree builder"); TreeBuilder builder(b); - for (auto & [name, child] : node.children) { - auto oid_p = std::get_if(&child.file); - auto oid = oid_p ? *oid_p : std::get(child.file).oid.value(); + for (const auto & [name, child] : node.children) { + const auto & oid = child.getOid(); if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode)) throw GitError("adding a file to a tree builder"); } @@ -1500,11 +1503,11 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink if (git_treebuilder_write(&oid, builder.get())) throw GitError("creating a tree object"); node.oid = oid; - }(_state.lock()->root); + }(root); repo->flush(); - return toHash(_state.lock()->root.oid.value()); + return toHash(root.oid.value()); } }; From 37a29c9483e160348668ffbc4bdc9fa0f833262d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 7 Jul 2026 21:18:08 +0300 Subject: [PATCH 318/364] libutil-test-support: Factor out SourceAccessor gmock matchers This is useful in more places. --- src/libfetchers-tests/git-utils.cc | 25 ++++-- src/libfetchers-tests/meson.build | 3 + .../include/nix/util/tests/gmock-matchers.hh | 61 +++++++++++++ src/libutil-tests/source-accessor.cc | 85 +++---------------- 4 files changed, 96 insertions(+), 78 deletions(-) diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 1d0300287e59..d36148b25174 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -86,13 +86,24 @@ TEST_F(GitUtilsTest, sink_basic) auto result = repo->dereferenceSingletonDirectory(sink->flush()); auto accessor = repo->getAccessor(result, {}, getRepoName()); - auto entries = accessor->readDirectory(CanonPath::root); - ASSERT_EQ(entries.size(), 5u); - ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); - ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); - ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); - ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0u); - ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); + + ASSERT_THAT( + accessor, + testing::HasDirectory( + CanonPath::root, + std::set{ + "hello", + "bye", + "bye-link", + "empty", + "links", + })); + + ASSERT_THAT(accessor, testing::HasContents(CanonPath("hello"), "hello world")); + ASSERT_THAT(accessor, testing::HasContents(CanonPath("bye"), "thanks for all the fish")); + ASSERT_THAT(accessor, testing::HasSymlink(CanonPath("bye-link"), "bye")); + ASSERT_THAT(accessor, testing::HasDirectory(CanonPath("empty"), std::set{})); + ASSERT_THAT(accessor, testing::HasContents(CanonPath("links/foo"), "hello world")); }; TEST_F(GitUtilsTest, sink_hardlink) diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 22cd6b378302..7123df54ebbe 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -33,6 +33,9 @@ deps_private += rapidcheck gtest = dependency('gtest', main : true) deps_private += gtest +gmock = dependency('gmock') +deps_private += gmock + libgit2 = dependency('libgit2') deps_private += libgit2 diff --git a/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh b/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh index e48faca7c68e..f440e8169587 100644 --- a/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh +++ b/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh @@ -2,6 +2,7 @@ ///@file #include "nix/util/error.hh" +#include "nix/util/source-accessor.hh" #include "nix/util/terminal.hh" #include @@ -66,4 +67,64 @@ inline auto ThrowsSysError(int expected) return ::testing::Throws(::testing::Field(&SysError::errNo, expected)); } +MATCHER_P2(HasContents, path, expected, "") +{ + auto stat = arg->maybeLstat(path); + if (!stat) { + *result_listener << arg->showPath(path) << " does not exist"; + return false; + } + if (stat->type != SourceAccessor::tRegular) { + *result_listener << arg->showPath(path) << " is not a regular file"; + return false; + } + auto actual = arg->readFile(path); + if (actual != expected) { + *result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual); + return false; + } + return true; +} + +MATCHER_P2(HasSymlink, path, target, "") +{ + auto stat = arg->maybeLstat(path); + if (!stat) { + *result_listener << arg->showPath(path) << " does not exist"; + return false; + } + if (stat->type != SourceAccessor::tSymlink) { + *result_listener << arg->showPath(path) << " is not a symlink"; + return false; + } + auto actual = arg->readLink(path); + if (actual != target) { + *result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual); + return false; + } + return true; +} + +MATCHER_P2(HasDirectory, path, dirents, "") +{ + auto stat = arg->maybeLstat(path); + if (!stat) { + *result_listener << arg->showPath(path) << " does not exist"; + return false; + } + if (stat->type != SourceAccessor::tDirectory) { + *result_listener << arg->showPath(path) << " is not a directory"; + return false; + } + auto actual = arg->readDirectory(path); + std::set actualKeys, expectedKeys(dirents.begin(), dirents.end()); + for (auto & [k, _] : actual) + actualKeys.insert(k); + if (actualKeys != expectedKeys) { + *result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys); + return false; + } + return true; +} + } // namespace nix::testing diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index 836c5895a285..fd281eb27f54 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -1,6 +1,7 @@ #include "nix/util/fs-sink.hh" #include "nix/util/file-system.hh" #include "nix/util/processes.hh" +#include "nix/util/tests/gmock-matchers.hh" #include #include @@ -8,66 +9,6 @@ namespace nix { -MATCHER_P2(HasContents, path, expected, "") -{ - auto stat = arg->maybeLstat(path); - if (!stat) { - *result_listener << arg->showPath(path) << " does not exist"; - return false; - } - if (stat->type != SourceAccessor::tRegular) { - *result_listener << arg->showPath(path) << " is not a regular file"; - return false; - } - auto actual = arg->readFile(path); - if (actual != expected) { - *result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual); - return false; - } - return true; -} - -MATCHER_P2(HasSymlink, path, target, "") -{ - auto stat = arg->maybeLstat(path); - if (!stat) { - *result_listener << arg->showPath(path) << " does not exist"; - return false; - } - if (stat->type != SourceAccessor::tSymlink) { - *result_listener << arg->showPath(path) << " is not a symlink"; - return false; - } - auto actual = arg->readLink(path); - if (actual != target) { - *result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual); - return false; - } - return true; -} - -MATCHER_P2(HasDirectory, path, dirents, "") -{ - auto stat = arg->maybeLstat(path); - if (!stat) { - *result_listener << arg->showPath(path) << " does not exist"; - return false; - } - if (stat->type != SourceAccessor::tDirectory) { - *result_listener << arg->showPath(path) << " is not a directory"; - return false; - } - auto actual = arg->readDirectory(path); - std::set actualKeys, expectedKeys(dirents.begin(), dirents.end()); - for (auto & [k, _] : actual) - actualKeys.insert(k); - if (actualKeys != expectedKeys) { - *result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys); - return false; - } - return true; -} - class FSSourceAccessorTest : public ::testing::Test { protected: @@ -105,17 +46,19 @@ TEST_F(FSSourceAccessorTest, works) sink.createSymlink(CanonPath("a/dirlink"), "../subdir"); } - EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), HasContents(CanonPath::root, "content1")); - EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), HasSymlink(CanonPath::root, "target")); + EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), testing::HasContents(CanonPath::root, "content1")); + EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), testing::HasSymlink(CanonPath::root, "target")); EXPECT_THAT( makeFSSourceAccessor(tmpDir), - HasDirectory(CanonPath::root, std::set{"file1", "subdir", "rootlink", "a"})); - EXPECT_THAT(makeFSSourceAccessor(tmpDir / "subdir"), HasDirectory(CanonPath::root, std::set{"file2"})); + testing::HasDirectory(CanonPath::root, std::set{"file1", "subdir", "rootlink", "a"})); + EXPECT_THAT( + makeFSSourceAccessor(tmpDir / "subdir"), + testing::HasDirectory(CanonPath::root, std::set{"file2"})); { auto accessor = makeFSSourceAccessor(tmpDir); - EXPECT_THAT(accessor, HasContents(CanonPath("file1"), "content1")); - EXPECT_THAT(accessor, HasContents(CanonPath("subdir/file2"), "content2")); + EXPECT_THAT(accessor, testing::HasContents(CanonPath("file1"), "content1")); + EXPECT_THAT(accessor, testing::HasContents(CanonPath("subdir/file2"), "content2")); EXPECT_TRUE(accessor->pathExists(CanonPath("file1"))); EXPECT_FALSE(accessor->pathExists(CanonPath("nonexistent"))); @@ -159,9 +102,9 @@ TEST_F(FSSourceAccessorTest, invalidateCacheDropsStaleDirFds) EXPECT_FALSE(accessor->pathExists(CanonPath("a/b/f"))); EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/g"))); - EXPECT_THAT(accessor, HasContents(CanonPath("a/b/g"), "new")); - EXPECT_THAT(accessor, HasDirectory(CanonPath("a/b"), (std::set{"g", "l"}))); - EXPECT_THAT(accessor, HasSymlink(CanonPath("a/b/l"), "g")); + EXPECT_THAT(accessor, testing::HasContents(CanonPath("a/b/g"), "new")); + EXPECT_THAT(accessor, testing::HasDirectory(CanonPath("a/b"), (std::set{"g", "l"}))); + EXPECT_THAT(accessor, testing::HasSymlink(CanonPath("a/b/l"), "g")); } /* ---------------------------------------------------------------------------- @@ -178,7 +121,7 @@ TEST_F(FSSourceAccessorTest, RestoreSinkRegularFileAtRoot) sink.createRegularFile(CanonPath::root, [](CreateRegularFileSink & crf) { crf("root content"); }); } - EXPECT_THAT(makeFSSourceAccessor(filePath), HasContents(CanonPath::root, "root content")); + EXPECT_THAT(makeFSSourceAccessor(filePath), testing::HasContents(CanonPath::root, "root content")); } TEST_F(FSSourceAccessorTest, RestoreSinkSymlinkAtRoot) @@ -194,7 +137,7 @@ TEST_F(FSSourceAccessorTest, RestoreSinkSymlinkAtRoot) sink.createSymlink(CanonPath::root, "symlink_target"); } - EXPECT_THAT(makeFSSourceAccessor(linkPath), HasSymlink(CanonPath::root, "symlink_target")); + EXPECT_THAT(makeFSSourceAccessor(linkPath), testing::HasSymlink(CanonPath::root, "symlink_target")); } } // namespace nix From 973f502b0267045a067295288659db954b59cbdb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 7 Jul 2026 21:35:51 +0300 Subject: [PATCH 319/364] libfetchers-tests: Add missing coverage for tarball unpacking special cases --- src/libfetchers-tests/git-utils.cc | 123 +++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index d36148b25174..c1b357e12a0a 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -190,6 +190,129 @@ TEST_F(GitUtilsTest, sink_no_parent_dir_hardlink) ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); } +TEST_F(GitUtilsTest, sink_replacing_empty_directory) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createDirectory(CanonPath("foo/bar")); + /* Under tarball unpacking semantics, creating the same directories + (implicitly or explicitly) is fine. */ + sink->createDirectory(CanonPath("foo/bar")); + sink->createDirectory(CanonPath("foo")); + + sink->createRegularFile(CanonPath("foo/bar"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + auto accessor = repo->getAccessor(sink->flush(), {}, getRepoName()); + + ASSERT_THAT(accessor, testing::HasDirectory(CanonPath("foo"), std::set{"bar"})); + ASSERT_THAT(accessor, testing::HasContents(CanonPath("foo/bar"), "test")); +} + +TEST_F(GitUtilsTest, sink_replacing_non_empty_directory) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createDirectory(CanonPath("foo/bar")); + + /* This fails. libarchive (and other tarball unpackers) doesn't recursive unlink existing non-empty + directories. + https://github.com/libarchive/libarchive/blob/761652401fe35fca9744607a0cf0009afbf04f42/libarchive/archive_write_disk_posix.c#L3411-L3417 + */ + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->flush(); + }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("cannot create 'foo', conflicting non-empty directory"))); +} + +TEST_F(GitUtilsTest, sink_hardlink_to_directory) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createHardlink(CanonPath("bar"), CanonPath("foo")); + + sink->flush(); + }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("cannot create a hard link to a directory"))); +} + +TEST_F(GitUtilsTest, sink_hardlink_to_directory_root) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createHardlink(CanonPath("bar"), CanonPath::root); + + auto accessor = repo->getAccessor(sink->flush(), {}, getRepoName()); + + /* FIXME: Why does it behave this way? This seems like a bug. */ + ASSERT_THAT( + accessor, + testing::HasDirectory( + CanonPath::root, + std::set{ + "foo", + })); +} + +TEST_F(GitUtilsTest, sink_hardlink_to_self) +{ + /* Here we are more strict than libarchive, which only warns on cyclic hardlinks. + https://github.com/libarchive/libarchive/blob/761652401fe35fca9744607a0cf0009afbf04f42/libarchive/archive_write_disk_posix.c#L632-L641 + */ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createHardlink(CanonPath("foo"), CanonPath("foo")); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("/foo"))); +} + +TEST_F(GitUtilsTest, sink_non_directory_root) +{ + /* FIXME: Allow non-directory roots. GitFileSystemObjectSink is too tarball-brained. */ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createRegularFile(CanonPath::root, [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->flush(); + }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("cannot create a file at the root of the git repository"))); +} + TEST_F(GitUtilsTest, peel_reference) { // Create a commit in the repo From 1736f6f01a53d8e03a978c744a7326420d3e2537 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 8 Jul 2026 03:41:02 +0300 Subject: [PATCH 320/364] libfetchers: Bring the packfile backend refresh hack back for the main repo Since we now also include *this repo in the set of writers, we need to bring this hack back - otherwise we still get the huge getdents storms. Also we need to manually refresh after all blobs are written to see the new packfiles. Also remove the TODO since that has been effectively addressed by writing symlinks and trees together to the same repo. --- src/libfetchers/git-utils.cc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 50df7861bcf7..9b433384d703 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -447,7 +447,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this */ Pool getPool() { - // TODO: as an optimization, it would be nice to include `this` in the pool. return Pool(std::numeric_limits::max(), [this]() -> ref { auto repo = make_ref(path, options); @@ -1194,6 +1193,13 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink ThreadPool workers{concurrency}; + /** + * If repo has a non-null packBackend, this has a copy of the refresh function + * from the backend virtual table. This is needed to restore it after we've flushed + * the sink. We modify it to avoid unnecessary I/O on non-existent oids. + */ + decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr; + /** Total file contents in flight. */ std::atomic totalBufSize{0}; @@ -1203,6 +1209,8 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink : repo(repo) , repoPool(repo->getPool()) { + if (auto * backend = repo->packBackend) + packfileOdbRefresh = std::exchange(backend->refresh, nullptr); } GitFileSystemObjectSinkImpl(GitFileSystemObjectSinkImpl &&) = delete; @@ -1215,6 +1223,8 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink // Make sure the worker threads are destroyed before any state // they're referring to. workers.shutdown(); + if (auto * backend = repo->packBackend; backend && packfileOdbRefresh) + backend->refresh = packfileOdbRefresh; } struct Child; @@ -1475,6 +1485,10 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink workers.process(); } + if (auto * backend = repo->packBackend) + /* We are done writing blobs. Need to refresh to get the objects written by other threads. */ + packfileOdbRefresh(backend); + // Write the Git trees to disk. Would be nice to have this multithreaded too, but that's hard because a tree // can't refer to an object that hasn't been written yet. Also it doesn't make a big difference for performance. @@ -1507,6 +1521,9 @@ struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink repo->flush(); + if (auto * backend = repo->packBackend) + backend->refresh = std::exchange(packfileOdbRefresh, nullptr); + return toHash(root.oid.value()); } }; From 3aff4dc5edf30998d64eec024de186ac2d6fb5ea Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 8 Jul 2026 23:56:21 +0300 Subject: [PATCH 321/364] Once again follow final symlinks in GitRepoImpl::getAccessor This used to be the case before 02e4f4ad75fda707a594ffd8ad511691fc1a311f, and this restores that compatibility. Ideally we'd check that access to that path is allowed, but libfetchers does this now pretty inconsistently. Also test a lot more things. --- src/libfetchers/git-utils.cc | 16 +++++++++------- tests/functional/git/meson.build | 5 ++++- tests/functional/git/symlinked-repo.sh | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 tests/functional/git/symlinked-repo.sh diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 286d31130e36..e61242eed5df 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1474,13 +1474,15 @@ ref GitRepoImpl::getAccessor( const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); - ref fileAccessor = AllowListSourceAccessor::create( - makeFSSourceAccessor(path), - /*allowedPrefixes=*/wd.files, - // Always allow access to the root, but not its children. - /*allowedPaths=*/{CanonPath::root}, - std::move(makeNotAllowedError)) - .cast(); + ref fileAccessor = + AllowListSourceAccessor::create( + // Follow the final symlink to the repo. Older nix versions used to do this (maybe somewhat accidentally). + makeFSSourceAccessor(path, /*trackLastModified=*/false, FinalSymlink::Follow), + /*allowedPrefixes=*/wd.files, + // Always allow access to the root, but not its children. + /*allowedPaths=*/{CanonPath::root}, + std::move(makeNotAllowedError)) + .cast(); if (options.exportIgnore) fileAccessor = make_ref(self, fileAccessor, std::nullopt); return fileAccessor; diff --git a/tests/functional/git/meson.build b/tests/functional/git/meson.build index af6882698b2c..73c4bc782871 100644 --- a/tests/functional/git/meson.build +++ b/tests/functional/git/meson.build @@ -1,6 +1,9 @@ suites += { 'name' : 'git', 'deps' : [], - 'tests' : [ 'packed-refs-no-cache.sh' ], + 'tests' : [ + 'packed-refs-no-cache.sh', + 'symlinked-repo.sh', + ], 'workdir' : meson.current_source_dir(), } diff --git a/tests/functional/git/symlinked-repo.sh b/tests/functional/git/symlinked-repo.sh new file mode 100644 index 000000000000..e30b0b3c0d8f --- /dev/null +++ b/tests/functional/git/symlinked-repo.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source ../common.sh + +requireGit + +repo=$TEST_ROOT/repo +link=$TEST_ROOT/link + +createGitRepo "$repo" +ln -s "$repo" "$link" + +# Dereferencing final symlink component should work and follow it to the directory. +# Also test various cases of symlink trickery in case that ever changes. +for path in {repo,link}{,/,/.} {repo,link}/.././repo{,/,/.}; do + [ "$(nix-instantiate --eval --expr "builtins.readFileType (builtins.fetchTree \"git+file://$TEST_ROOT/$path\").outPath")" == '"directory"' ] +done From 9640ab1fa4a88def1c5be31e45cd819caa24a638 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 11 Jul 2026 00:26:17 +0300 Subject: [PATCH 322/364] libutil: Don't trample parent's havePrivateMountNs in restoreMountNamespace This is only a band-aid, see the comment for the explanation why the current code sucks in an indescribable way. A much better solution would be to get rid of vfork() entirely (or pay the incredible cost of making it safe for each libc we target) by replacing that with a zygote process. That's a huge change though, so let's cross our fingers and pray that this sucky code won't lead to a security issue. --- src/libutil/linux/linux-namespaces.cc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index f0416307d126..5ad9bfc9decc 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -161,12 +161,25 @@ void remountReadOnlyWritable(const std::filesystem::path & path) throw SysError("remounting %s writable", PathFmt(path)); } +/* This code runs in a (possibly) vfork-ed child, so technically everything you see below is beyond + broken because vfork()-ed child: + + * Must not trample parent's memory in any way shape or form. That includes (but not limited to) + * Throwing any exceptions (because that would unwind into the parent stack frame and do who knows what). + * Modify any state - obviously that includes global state. + * Not allocate any memory, since that can also lead to a deadlock if some thread in the (now stopped) parent + holds a lock while we are running. That's because *all* of the parent tasks are suspended for the duration + of the vfork. + + As it stands now, this code should be considered incredibly fragile and slated for a complete rework. + */ void restoreMountNamespace() { if (!havePrivateMountNs) return; try { + /* FIXME: Allocation in a possibly vforked child. */ auto savedCwd = std::filesystem::current_path(); if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) @@ -182,7 +195,8 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); - havePrivateMountNs = false; + /* Do not reset havePrivateMountNs! This code can run in a vfork-ed child and we absolutely + must not trample any of the parent's state. */ } catch (Error & e) { debug(e.msg()); } From f7495a36b7ffd38b9832f146bcccb13537f2c0c3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 12 Jul 2026 00:35:40 +0300 Subject: [PATCH 323/364] libutil: Fix Config::toKeyValue This was accidentally broken in 450e5ec6185e2e1102e67ec7a348a0dc8955692d. It's not used anywhere currently now though. --- src/libutil-tests/config.cc | 8 ++++++++ src/libutil/configuration.cc | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index b9eec0c4175f..d7b024ead170 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -325,4 +325,12 @@ TEST(Config, applyConfigInvalidThrows) ASSERT_THROW(config.applyConfig("value == key"), UsageError); ASSERT_THROW(config.applyConfig("value "), UsageError); } + +TEST(Config, toKeyValue) +{ + Config config; + Setting foo{&config, "default", "name-of-the-setting", "description", {"alias-of-the-setting"}}; + ASSERT_EQ(config.toKeyValue(), "name-of-the-setting = default\n"); +} + } // namespace nix diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index 39e21fc0f80f..b84867885098 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -216,7 +216,7 @@ std::string Config::toKeyValue() { std::string res; for (const auto & s : _settings) - if (s.second.isAlias) + if (!s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; } From 98a3bcd44b721d91436ae7d56511ef1558737fff Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 12 Jul 2026 01:02:04 +0300 Subject: [PATCH 324/364] libutil-tests: Factor out CaptureLogging, make more tests capture warning output This is certainly useful in more places. libutil-tests was being too loud with warnings, so that's my motivation for doing this. Ideally we'd test that the intended warnings are produced too, but that can be done in a follow-up. --- src/libexpr-tests/primops.cc | 44 +------------ .../include/nix/util/tests/capture-logging.hh | 61 +++++++++++++++++++ .../include/nix/util/tests/meson.build | 1 + src/libutil-tests/url.cc | 33 +++++++--- 4 files changed, 89 insertions(+), 50 deletions(-) create mode 100644 src/libutil-test-support/include/nix/util/tests/capture-logging.hh diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index 08f610ced0f4..fef732e1e3f5 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -4,47 +4,10 @@ #include "nix/expr/eval-settings.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/tests/capture-logging.hh" #include "nix/expr/tests/libexpr.hh" namespace nix { -class CaptureLogger : public Logger -{ - std::ostringstream oss; - -public: - CaptureLogger() {} - - std::string get() const - { - return oss.str(); - } - - void log(Verbosity lvl, std::string_view s) override - { - oss << s << std::endl; - } - - void logEI(const ErrorInfo & ei) override - { - showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - } -}; - -class CaptureLogging -{ - Logger * oldLogger; -public: - CaptureLogging() - { - oldLogger = logger; - logger = new CaptureLogger(); - } - - ~CaptureLogging() - { - logger = oldLogger; - } -}; // Testing eval of PrimOp's class PrimOpTest : public LibExprTest @@ -141,11 +104,10 @@ TEST_F(PrimOpTest, deepSeq) TEST_F(PrimOpTest, trace) { - CaptureLogging l; + testing::CaptureLogging l; auto v = eval("builtins.trace \"test string 123\" 123"); ASSERT_THAT(v, IsIntEq(123)); - auto text = (dynamic_cast(logger))->get(); - ASSERT_NE(text.find("test string 123"), std::string::npos); + ASSERT_THAT(l.get(), ::testing::HasSubstr("test string 123")); } TEST_F(PrimOpTest, placeholder) diff --git a/src/libutil-test-support/include/nix/util/tests/capture-logging.hh b/src/libutil-test-support/include/nix/util/tests/capture-logging.hh new file mode 100644 index 000000000000..758fdd524ead --- /dev/null +++ b/src/libutil-test-support/include/nix/util/tests/capture-logging.hh @@ -0,0 +1,61 @@ +#pragma once + +#include "nix/util/logging.hh" + +#include + +namespace nix::testing { + +class CaptureLogger : public Logger +{ + std::ostringstream oss; + +public: + CaptureLogger() {} + + std::string get() const + { + return oss.str(); + } + + void log(Verbosity lvl, std::string_view s) override + { + oss << s << std::endl; + } + + void logEI(const ErrorInfo & ei) override + { + showErrorInfo(oss, ei, loggerSettings.showTrace.get()); + } +}; + +class CaptureLogging +{ + std::unique_ptr logger; + Logger * oldLogger; + +public: + CaptureLogging() + { + oldLogger = nix::logger; + logger = std::make_unique(); + nix::logger = logger.get(); + } + + std::string get() const + { + return logger->get(); + } + + CaptureLogging(CaptureLogging &&) = delete; + CaptureLogging(const CaptureLogging &) = delete; + CaptureLogging & operator=(CaptureLogging &&) = delete; + CaptureLogging & operator=(const CaptureLogging &) = delete; + + ~CaptureLogging() + { + nix::logger = oldLogger; + } +}; + +} // namespace nix::testing diff --git a/src/libutil-test-support/include/nix/util/tests/meson.build b/src/libutil-test-support/include/nix/util/tests/meson.build index 9f09183f33f9..b084c9f5ad4b 100644 --- a/src/libutil-test-support/include/nix/util/tests/meson.build +++ b/src/libutil-test-support/include/nix/util/tests/meson.build @@ -3,6 +3,7 @@ include_dirs = [ include_directories('../../..') ] headers = files( + 'capture-logging.hh', 'characterization.hh', 'gmock-matchers.hh', 'gtest-with-params.hh', diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 042572b6fe40..491a3a4f41f3 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -1,9 +1,12 @@ #include "nix/util/url.hh" +#include "nix/util/tests/capture-logging.hh" #include "nix/util/tests/gmock-matchers.hh" + #include #include #include +#include namespace nix { @@ -25,7 +28,19 @@ std::ostream & operator<<(std::ostream & os, const FixGitURLParam & param) } class FixGitURLTestSuite : public ::testing::TestWithParam -{}; +{ + std::optional captureLogging; + + void SetUp() override + { + captureLogging.emplace(); + } + + void TearDown() override + { + captureLogging.reset(); + } +}; INSTANTIATE_TEST_SUITE_P( FixGitURLs, @@ -366,7 +381,7 @@ TEST_P(FixGitURLTestSuite, parsedNormalized) EXPECT_EQ(actual.to_string(), p.expected); } -TEST(FixGitURLTestSuite, rejectFileURLWithAuthority) +TEST_F(FixGitURLTestSuite, rejectFileURLWithAuthority) { /* From the underlying `parseURL` validations. */ EXPECT_THAT( @@ -375,7 +390,7 @@ TEST(FixGitURLTestSuite, rejectFileURLWithAuthority) testing::HasSubstrIgnoreANSIMatcher("file:// URL 'file://var/repos/x' has unexpected authority 'var'"))); } -TEST(FixGitURLTestSuite, rejectRelativePath) +TEST_F(FixGitURLTestSuite, rejectRelativePath) { /* From the underlying `parseURL` validations. */ EXPECT_THAT( @@ -383,7 +398,7 @@ TEST(FixGitURLTestSuite, rejectRelativePath) ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("doesn't have a scheme"))); } -TEST(FixGitURLTestSuite, rejectEmptyPathGitScp) +TEST_F(FixGitURLTestSuite, rejectEmptyPathGitScp) { /* Reject SCP-style URLs with no path component. */ EXPECT_THAT( @@ -392,7 +407,7 @@ TEST(FixGitURLTestSuite, rejectEmptyPathGitScp) testing::HasSubstrIgnoreANSIMatcher("SCP-style Git URL 'host:' has an empty path"))); } -TEST(FixGitURLTestSuite, rejectMalformedBracketedURLs) +TEST_F(FixGitURLTestSuite, rejectMalformedBracketedURLs) { /* Brackets not in host position go through the colon-based path, consistent with git (which also finds the first colon). These @@ -418,7 +433,7 @@ TEST(FixGitURLTestSuite, rejectMalformedBracketedURLs) EXPECT_EQ(parsed3.authority->user, "user:"); } -TEST(FixGitURLTestSuite, mismatchedBrackets) +TEST_F(FixGitURLTestSuite, mismatchedBrackets) { /* Missing `]`: git's `host_end()` falls back to `end = host` and finds the first `:` as separator. We do the same — fall through @@ -454,7 +469,7 @@ TEST(FixGitURLTestSuite, mismatchedBrackets) ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("is not a valid IPv6 address"))); } -TEST(FixGitURLTestSuite, slashBeforeColonIsNotScp) +TEST_F(FixGitURLTestSuite, slashBeforeColonIsNotScp) { /* A slash before the first colon means it's not SCP — consistent with git's `url_is_local_not_ssh()`. */ @@ -463,7 +478,7 @@ TEST(FixGitURLTestSuite, slashBeforeColonIsNotScp) ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("doesn't have a scheme"))); } -TEST(FixGitURLTestSuite, noColonIsNotScp) +TEST_F(FixGitURLTestSuite, noColonIsNotScp) { /* No `:` at all means not SCP — consistent with git's `url_is_local_not_ssh()` in `connect.c`. @@ -479,7 +494,7 @@ TEST(FixGitURLTestSuite, noColonIsNotScp) ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("is not a valid URL"))); } -TEST(FixGitURLTestSuite, gitBugDiscardedCharsBetweenBracketAndColon) +TEST_F(FixGitURLTestSuite, gitBugDiscardedCharsBetweenBracketAndColon) { /* Git's `host_end()` returns `end` past `]`, then `strchr(end, ':')` finds `:` anywhere after — silently discarding characters between From b8a98cc58b533ea87ab3058035a26d45ff3fcadb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 12 Jul 2026 01:09:51 +0300 Subject: [PATCH 325/364] libutil-tests: Fix compilation warning in CompressionDecompressionTest Silences a warning about a dangling else inside the macro. --- src/libutil-tests/compression.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil-tests/compression.cc b/src/libutil-tests/compression.cc index 7978a008fe65..60d2cd29ed35 100644 --- a/src/libutil-tests/compression.cc +++ b/src/libutil-tests/compression.cc @@ -29,8 +29,9 @@ TEST_P(CompressionDecompressionTest, roundtrip) TEST_P(CompressionDecompressionTest, empty) { auto compressed = compress(GetParam(), ""); - if (GetParam() != CompressionAlgo::none) + if (GetParam() != CompressionAlgo::none) { ASSERT_FALSE(compressed.empty()); + } auto o = decompress(GetParam(), compressed); ASSERT_EQ(o, ""); } From 526e916abd12032141f7bd45d67d2d9f252b1be3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 12 Jul 2026 15:29:17 +0300 Subject: [PATCH 326/364] packaging: Fix sqlite on mingw tcl is broken on mingw in nixpkgs and so is sqlite. This makes it build now at least. --- packaging/dependencies.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index c8700fffa38e..db7948566f72 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -90,6 +90,17 @@ scope: { ; }; + sqlite = + if !stdenv.hostPlatform.isWindows then + pkgs.sqlite + else + pkgs.sqlite.overrideAttrs (prevAttrs: { + nativeBuildInputs = lib.filter (x: !(x.pname == "tcl")) prevAttrs.nativeBuildInputs or [ ]; + configureFlags = (lib.filter (x: !(lib.hasPrefix "--with-tcl" x)) prevAttrs.configureFlags) ++ [ + "--disable-tcl" + ]; + }); + libgit2 = if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then pkgs.libgit2 From 368d5b38e63fa410485a600f9973feb681eb70c3 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 13 Jul 2026 15:28:53 -0400 Subject: [PATCH 327/364] Add rust installer to hydra release-jobs Signed-off-by: Lisanna Dettwyler --- packaging/release-jobs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index f82c23c9c1ac..28ee0cb4e00b 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -51,6 +51,7 @@ let binaryTarballCross installerScript installerScriptForGHA + rustInstaller dockerImage ; From 5063db70674d5db052af50d59135aa9ae03c16ba Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 00:36:16 +0300 Subject: [PATCH 328/364] Don't openStore in nix-env --version This has caused dealocks after the fixes to acquire an exclusive lock for performing store DB migrations, because upgrade-nix.cc runs a nix-env --version to check that the command works at all. Since the old code would try to open a store, we'd deadlock. Fixes #16136 --- src/libmain/include/nix/main/shared.hh | 2 +- src/nix/nix-env/nix-env.cc | 28 ++++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index f9e771205ba2..7b0e7a54d4e0 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -26,7 +26,7 @@ void parseCmdLine( const Strings & args, fun parseArg); -void printVersion(const std::string & programName); +[[noreturn]] void printVersion(const std::string & programName); /** * Ugh. No better place to put this. diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 6663cb2db67a..8568b1c8d59c 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -1391,7 +1391,7 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr } } -static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) +[[noreturn]] static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) { printVersion("nix-env"); } @@ -1508,23 +1508,25 @@ static int main_nix_env(int argc, char ** argv) if (!op) throw UsageError("no operation specified"); - auto store = openStore(); + if (op != opVersion) { + auto store = openStore(); - globals.state = - std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); - globals.state->repair = myArgs.repair; + globals.state = + std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); + globals.state->repair = myArgs.repair; - globals.instSource.nixExprPath = std::make_shared( - file != "" ? lookupFileArg(*globals.state, file) - : globals.state->rootPath(CanonPath(nixExprPath.string()))); + globals.instSource.nixExprPath = std::make_shared( + file != "" ? lookupFileArg(*globals.state, file) + : globals.state->rootPath(CanonPath(nixExprPath.string()))); - globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); - if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE").value_or(""); + if (globals.profile == "") + globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") - globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); + if (globals.profile == "") + globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); + } op(globals, std::move(opFlags), std::move(opArgs)); From c1cc521d2ffbc0b9e0a2cf3e3aa22c9d04f037ef Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 13 Jul 2026 18:18:10 -0400 Subject: [PATCH 329/364] Revert temproots path name change This is causing breakages when downgrading from nix 2.35.0, since it expects the filename to have a parseable integer. Signed-off-by: Lisanna Dettwyler --- src/libstore/gc.cc | 4 ++-- src/libstore/local-store.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e399dd949a61..8e997c8e2471 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -61,8 +61,8 @@ void LocalStore::createTempRootsFile() while (1) { if (pathExists(fnTempRoots)) - /* The file is stale since each LocalStore instance - uses a unique filename (pid + counter). */ + /* It *must* be stale, since there can be no two + processes with the same pid. */ tryUnlink(fnTempRoots); *fdTempRoots = openLockFile(fnTempRoots, true); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a1c161c0739..909f22369dc0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -129,7 +129,7 @@ LocalStore::LocalStore(ref config) , reservedPath(dbDir / "reserved") , schemaPath(dbDir / "schema") , tempRootsDir(config->stateDir.get() / "temproots") - , fnTempRoots(makeTempPath(tempRootsDir, "temproots")) + , fnTempRoots(tempRootsDir / std::to_string(getpid())) { auto state(_state->lock()); state->stmts = std::make_unique(); From f553163d4cbad6d437b8ee12d660cb5a96df1e99 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 01:46:38 +0300 Subject: [PATCH 330/364] Bump pinned nix in install-nix-action composite action --- .github/actions/install-nix-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index f889944b2d03..2fb433b741c3 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -9,7 +9,7 @@ inputs: install_url: description: "URL of the Nix installer" required: false - default: "https://releases.nixos.org/nix/nix-2.32.1/install" + default: "https://releases.nixos.org/nix/nix-2.34.8/install" github_token: description: "Github token" required: true From 6109492eda3e89d1e01fa8da740afdfcf75a481d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Jul 2026 18:08:19 +0300 Subject: [PATCH 331/364] libutil: Make more AutoCloseFD methods noexcept --- src/libutil/file-descriptor.cc | 6 +++--- src/libutil/include/nix/util/file-descriptor.hh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index f5f5ee05a69d..469220d4c650 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -257,7 +257,7 @@ AutoCloseFD::~AutoCloseFD() } } -Descriptor AutoCloseFD::get() const +Descriptor AutoCloseFD::get() const noexcept { return fd; } @@ -288,12 +288,12 @@ void AutoCloseFD::startFsync() const #endif } -AutoCloseFD::operator bool() const +AutoCloseFD::operator bool() const noexcept { return fd != INVALID_DESCRIPTOR; } -Descriptor AutoCloseFD::release() +Descriptor AutoCloseFD::release() noexcept { Descriptor oldFD = fd; fd = INVALID_DESCRIPTOR; diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index a6796f22691e..53faa7acd8c7 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -264,9 +264,9 @@ public: AutoCloseFD & operator=(const AutoCloseFD & fd) = delete; // NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw because of close() AutoCloseFD & operator=(AutoCloseFD && fd); - Descriptor get() const; - explicit operator bool() const; - Descriptor release(); + Descriptor get() const noexcept; + explicit operator bool() const noexcept; + Descriptor release() noexcept; void close(); /** From cdbeb1c0cb0e9952f051064ec4cfdb1f2a7f196c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Jul 2026 00:19:29 +0300 Subject: [PATCH 332/364] libutil: Get rid of unsafe vfork() usage in runProgram2 Current usage of vfork() is definitely not async-signal-safe and does a lot of weird stuff like calling out to libc functions/throw exceptions and don't take care not to trample parent's state. Reclaiming performance should be done by carefully reimplementing this logic, borrowing experience from go/jvm/python runtimes that actually implement this safely. --- .../build/derivation-building-goal.cc | 1 - src/libutil/include/nix/util/processes.hh | 1 - src/libutil/linux/linux-namespaces.cc | 16 ---------- src/libutil/unix/processes.cc | 30 +++++-------------- src/nix/unix/daemon.cc | 1 - 5 files changed, 8 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 5531f984b966..8822eaff8b1b 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -1123,7 +1123,6 @@ static std::unique_ptr runPostBuildHook( hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); ProcessOptions processOptions; - processOptions.allowVfork = false; state->pid = startProcess( [&] { diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 64053fab4935..e1deafeda123 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -97,7 +97,6 @@ struct ProcessOptions std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; - bool allowVfork = false; /** * use clone() with the specified flags (Linux only) */ diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 5ad9bfc9decc..816acfba6fcb 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -161,25 +161,12 @@ void remountReadOnlyWritable(const std::filesystem::path & path) throw SysError("remounting %s writable", PathFmt(path)); } -/* This code runs in a (possibly) vfork-ed child, so technically everything you see below is beyond - broken because vfork()-ed child: - - * Must not trample parent's memory in any way shape or form. That includes (but not limited to) - * Throwing any exceptions (because that would unwind into the parent stack frame and do who knows what). - * Modify any state - obviously that includes global state. - * Not allocate any memory, since that can also lead to a deadlock if some thread in the (now stopped) parent - holds a lock while we are running. That's because *all* of the parent tasks are suspended for the duration - of the vfork. - - As it stands now, this code should be considered incredibly fragile and slated for a complete rework. - */ void restoreMountNamespace() { if (!havePrivateMountNs) return; try { - /* FIXME: Allocation in a possibly vforked child. */ auto savedCwd = std::filesystem::current_path(); if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) @@ -194,9 +181,6 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); - - /* Do not reset havePrivateMountNs! This code can run in a vfork-ed child and we absolutely - must not trample any of the parent's state. */ } catch (Error & e) { debug(e.msg()); } diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 32d59535883a..ea5f3c869320 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -207,17 +207,9 @@ void killUser(uid_t uid) using ChildWrapperFunction = fun; -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) __attribute__((noinline)); - -static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) +static pid_t doFork(ChildWrapperFunction & fun) { -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else pid_t pid = fork(); -#endif if (pid != 0) return pid; fun(); @@ -237,14 +229,12 @@ pid_t startProcess(fun processMain, const ProcessOptions & options) { auto newLogger = makeSimpleLogger().release(); ChildWrapperFunction wrapper = [&] { - if (!options.allowVfork) { - /* Set a simple logger, while leaking (not destroying) - the parent logger. We don't want to run the parent - logger's destructor since that will crash (e.g. when - ~ProgressBar() tries to join a thread that doesn't - exist. */ - logger = newLogger; - } + /* Set a simple logger, while leaking (not destroying) + the parent logger. We don't want to run the parent + logger's destructor since that will crash (e.g. when + ~ProgressBar() tries to join a thread that doesn't + exist. */ + logger = newLogger; try { #ifdef __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -284,7 +274,7 @@ pid_t startProcess(fun processMain, const ProcessOptions & options) throw Error("clone flags are only supported on Linux"); #endif } else - pid = doFork(options.allowVfork, wrapper); + pid = doFork(wrapper); if (pid == -1) throw SysError("unable to fork"); @@ -318,10 +308,6 @@ void runProgram2(const RunOptions & options) out.create(); ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; auto suspension = logger->suspendIf(options.isInteractive); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 956adc43c99b..9023e2be04d0 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -354,7 +354,6 @@ static void daemonLoop( options.errorPrefix = "unexpected Nix daemon error: "; options.dieWithParent = false; options.runExitHandlers = true; - options.allowVfork = false; startProcess( [&, storeConfig, closeListeners = std::move(closeListeners)]() { closeListeners(); From a47b1d8d2c85c404531bbb4d5f9e12a24916bbd3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Jul 2026 19:03:01 +0300 Subject: [PATCH 333/364] libutil: Split runProgram2 from unix/processes.cc to linux/processes.cc This way we have a way to extend/specialise this on Linux (possibly other platforms too) to get vfork()-based runProgram2 that's safer. It already has some linux-specific bits anyhow around restoreProcessContext and mount namespaces. --- src/libutil/linux/meson.build | 1 + src/libutil/linux/processes.cc | 76 ++++++++++++++++++++++++++++++++++ src/libutil/unix/processes.cc | 4 ++ 3 files changed, 81 insertions(+) create mode 100644 src/libutil/linux/processes.cc diff --git a/src/libutil/linux/meson.build b/src/libutil/linux/meson.build index b8053a5bb037..3f48646d2216 100644 --- a/src/libutil/linux/meson.build +++ b/src/libutil/linux/meson.build @@ -1,6 +1,7 @@ sources += files( 'cgroup.cc', 'linux-namespaces.cc', + 'processes.cc', ) subdir('include/nix/util') diff --git a/src/libutil/linux/processes.cc b/src/libutil/linux/processes.cc new file mode 100644 index 000000000000..cc4687778da8 --- /dev/null +++ b/src/libutil/linux/processes.cc @@ -0,0 +1,76 @@ +#include "nix/util/processes.hh" +#include "nix/util/current-process.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/signals.hh" +#include "nix/util/util.hh" + +#include +#include +#include +#include + +namespace nix { + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + /* Create a pipe. */ + Pipe out; + if (options.standardOut) + out.create(); + + ProcessOptions processOptions; + + auto suspension = logger->suspendIf(options.isInteractive); + + /* Fork. */ + Pid pid = startProcess( + [&] { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program.native()); + + restoreProcessContext(); + + if (options.lookupPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing %s", PathFmt(options.program)); + }, + processOptions); + + out.writeSide.close(); + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + if (status) + throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); +} + +} // namespace nix diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index ea5f3c869320..522e36298f74 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -298,6 +298,8 @@ std::string runProgram(std::filesystem::path program, bool lookupPath, const OsS return res.second; } +#ifndef __linux__ + void runProgram2(const RunOptions & options) { checkInterrupt(); @@ -359,6 +361,8 @@ void runProgram2(const RunOptions & options) throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } +#endif // __linux__ + ////////////////////////////////////////////////////////////////////// std::string statusToString(int status) From 5efe9212a130071e0496664324ce9d6f2853387e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Jul 2026 01:38:52 +0300 Subject: [PATCH 334/364] libutil: Move ensureStackSizeAtLeast and savedStackSize to unix specific files Exposing unix::savedStackSize in a private header (not installed) is intentional and will be required for making use of in in vfork() based runProgram2. --- src/libutil/current-process.cc | 46 +++------------------ src/libutil/unix/current-process-private.hh | 11 +++++ src/libutil/unix/current-process.cc | 41 ++++++++++++++++++ 3 files changed, 58 insertions(+), 40 deletions(-) create mode 100644 src/libutil/unix/current-process-private.hh diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index acb6e52dd733..a235d6b58bcf 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -8,6 +8,10 @@ #include "nix/util/environment-variables.hh" #include +#ifndef _WIN32 +# include "unix/current-process-private.hh" +#endif + #ifdef __APPLE__ # include #endif @@ -55,44 +59,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -#ifndef _WIN32 -size_t savedStackSize = 0; - -void ensureStackSizeAtLeast(size_t stackSize) -{ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast(limit.rlim_cur) < stackSize) { - savedStackSize = limit.rlim_cur; - if (limit.rlim_max < static_cast(stackSize)) { - if (getEnv("_NIX_TEST_NO_ENVIRONMENT_WARNINGS") != "1") { - logger->log( - lvlWarn, - HintFmt( - "Stack size hard limit is %1%, which is less than the desired %2%. If possible, increase the hard limit, e.g. with 'ulimit -Hs %3%'.", - limit.rlim_max, - stackSize, - stackSize / 1024) - .str()); - } - } - auto requestedSize = std::min(static_cast(stackSize), limit.rlim_max); - limit.rlim_cur = requestedSize; - if (setrlimit(RLIMIT_STACK, &limit) != 0) { - logger->log( - lvlError, - HintFmt( - "Failed to increase stack size from %1% to %2% (desired: %3%, maximum allowed: %4%): %5%", - savedStackSize, - requestedSize, - stackSize, - limit.rlim_max, - std::strerror(errno)) - .str()); - } - } -} -#endif - void restoreProcessContext(bool restoreMounts) { #ifndef _WIN32 @@ -105,10 +71,10 @@ void restoreProcessContext(bool restoreMounts) } #ifndef _WIN32 - if (savedStackSize) { + if (unix::savedStackSize) { struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; + limit.rlim_cur = unix::savedStackSize; setrlimit(RLIMIT_STACK, &limit); } } diff --git a/src/libutil/unix/current-process-private.hh b/src/libutil/unix/current-process-private.hh new file mode 100644 index 000000000000..c5ff790c130f --- /dev/null +++ b/src/libutil/unix/current-process-private.hh @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace nix { +namespace unix { + +extern size_t savedStackSize; + +} // namespace unix +} // namespace nix diff --git a/src/libutil/unix/current-process.cc b/src/libutil/unix/current-process.cc index eaa2424abcd9..3f0c8180356d 100644 --- a/src/libutil/unix/current-process.cc +++ b/src/libutil/unix/current-process.cc @@ -1,5 +1,10 @@ #include "nix/util/current-process.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/error.hh" +#include "nix/util/logging.hh" + +#include "unix/current-process-private.hh" + #include #include @@ -20,4 +25,40 @@ std::chrono::microseconds getCpuUserTime() return seconds + microseconds; } +size_t unix::savedStackSize = 0; + +void ensureStackSizeAtLeast(size_t stackSize) +{ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast(limit.rlim_cur) < stackSize) { + unix::savedStackSize = limit.rlim_cur; + if (limit.rlim_max < static_cast(stackSize)) { + if (getEnv("_NIX_TEST_NO_ENVIRONMENT_WARNINGS") != "1") { + logger->log( + lvlWarn, + HintFmt( + "Stack size hard limit is %1%, which is less than the desired %2%. If possible, increase the hard limit, e.g. with 'ulimit -Hs %3%'.", + limit.rlim_max, + stackSize, + stackSize / 1024) + .str()); + } + } + auto requestedSize = std::min(static_cast(stackSize), limit.rlim_max); + limit.rlim_cur = requestedSize; + if (setrlimit(RLIMIT_STACK, &limit) != 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (desired: %3%, maximum allowed: %4%): %5%", + unix::savedStackSize, + requestedSize, + stackSize, + limit.rlim_max, + std::strerror(errno)) + .str()); + } + } +} + } // namespace nix From 646ddf5e9dc944f20eafbba61066b073637506e4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jul 2026 20:34:55 +0200 Subject: [PATCH 335/364] doc(glossary): define hermetic, pinning, locking --- doc/manual/source/glossary.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index 112b0e3f8c0d..5ea6dae743a0 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -169,6 +169,11 @@ A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. +- [hermetic]{#gloss-hermetic} + + An evaluation or build process is hermetic when one can mechanically identify the set of all inputs that may affect it. + At the build level this is achieved by sandboxing; at the evaluation level by restricting impure access (as in [pure evaluation](@docroot@/command-ref/conf-file.md#conf-pure-eval)) together with [locking](#gloss-locking) or [pinning](#gloss-pinning) of the fetched inputs, taken transitively over pure fetches. + - [IFD]{#gloss-ifd} [Import From Derivation](./language/import-from-derivation.md) @@ -201,6 +206,10 @@ [instantiate]: #gloss-instantiate +- [locking]{#gloss-locking} + + In package management, *locking* is the concept or process of creating a lock file, which maps each mutable evaluation input to an immutable reference, so that future evaluations resolve to the same immutable versions rather than whatever the mutable references currently point to. + - [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix @@ -270,6 +279,16 @@ [package]: #package +- [pinning]{#gloss-pinning} + + Like [locking](#gloss-locking), but a pin only locks a single input. + A pinning solution may manage a collection of pins, + but serves the bottom-up purpose of fixing an input's reference on demand, + whereas locking implies a top down approach where all pins are "coerced" into a single place. + This "coercion" is generally achieved by means of high-level mechanisms such as programming language module systems. + Nix does not have such a restrictive module system, as even a flake can use expressions that pin or lock on their own. + It does not rely on the lock being total, but on a transitive fetching property; see [hermeticity](#gloss-hermetic). + - [profile]{#gloss-profile} A symlink to the current *user environment* of a user, e.g., From c6c3832c3aeab1656bd713050eb7927fc59cb92f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jul 2026 20:39:38 +0200 Subject: [PATCH 336/364] doc(pure-eval): define pure fetch --- doc/manual/source/glossary.md | 2 +- src/libexpr/include/nix/expr/eval-settings.hh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index 5ea6dae743a0..188e384772b5 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -172,7 +172,7 @@ - [hermetic]{#gloss-hermetic} An evaluation or build process is hermetic when one can mechanically identify the set of all inputs that may affect it. - At the build level this is achieved by sandboxing; at the evaluation level by restricting impure access (as in [pure evaluation](@docroot@/command-ref/conf-file.md#conf-pure-eval)) together with [locking](#gloss-locking) or [pinning](#gloss-pinning) of the fetched inputs, taken transitively over pure fetches. + At the build level this is achieved by sandboxing; at the evaluation level by restricting impure access (as in [pure evaluation](@docroot@/command-ref/conf-file.md#conf-pure-eval)) together with [locking](#gloss-locking) or [pinning](#gloss-pinning) of the fetched inputs, taken transitively over [pure fetches](@docroot@/command-ref/conf-file.md#pure-fetch). - [IFD]{#gloss-ifd} diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index d8a1fb97362f..33ae8509970f 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -202,6 +202,10 @@ public: - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime) - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) + + As a result, every fetch must be a []{#pure-fetch}*pure fetch* — one that references immutable content: + [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) and [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) require a locked revision, and [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) and [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) require a `sha256` hash. + A mutable reference, such as a Git branch or tag without a revision, is rejected, since its result could otherwise change over time. )"}; Setting traceImportFromDerivation{ From 8115a38536e21430d37d2915fa19ba7811c991ff Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jul 2026 20:42:27 +0200 Subject: [PATCH 337/364] doc(builtins): recommend lib (#13288) --- doc/manual/source/language/builtins-prefix.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/manual/source/language/builtins-prefix.md b/doc/manual/source/language/builtins-prefix.md index 8dd929be3601..ed78366b094a 100644 --- a/doc/manual/source/language/builtins-prefix.md +++ b/doc/manual/source/language/builtins-prefix.md @@ -28,6 +28,25 @@ Some built-ins are also exposed directly in the global scope: - [`toString`](#builtins-toString) - [`true`](#builtins-true) + + +> **Tip** +> +> **Should I use `builtins` or `lib`?** +> +> The built-ins are designed to be a stable interface that expressions can depend on over time, +> so that, for instance, old Nixpkgs versions continue to evaluate reproducibly. +> +> On the flip side, this means they have accumulated a few quirks that Nix is unable to change, +> but a library like Nixpkgs `lib` *can* improve, replace or deprecate those behaviors, +> because its sources are pinned where reproducibility matters. +> +> So while it is not wrong to use `builtins` directly, +> for instance in small Nixpkgs-independent projects, +> you will have a better experience using a library like `lib` as your primary source of functions, +> as it hides problematic functions, fixes up others, +> and helps you improve your code by means of future deprecations, which are still sufficiently rare. +
derivation attrs

derivation is described in From 07cefe5b38f45c301ffa2937f59083cbfc5db537 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Jul 2026 22:21:16 +0200 Subject: [PATCH 338/364] Reserve a couple of worker operation codes --- src/libstore/include/nix/store/worker-protocol.hh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index d09d30421dcf..96866a14f94f 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -256,6 +256,9 @@ enum struct WorkerProto::Op : uint64_t { AddBuildLog = 45, BuildPathsWithResults = 46, AddPermRoot = 47, + // QueryActiveBuilds = 48, // reserved for https://github.com/NixOS/nix/pull/15979 + // AddTempRoots = 49, // reserved for https://github.com/NixOS/nix/pull/16113 + // QueryPathInfos = 50, // reserved for https://github.com/DeterminateSystems/nix-src/pull/539 }; struct WorkerProto::ClientHandshakeInfo From d44960fd45abdf2579b069c0c43cd58e5de07eee Mon Sep 17 00:00:00 2001 From: qsxDree Date: Thu, 16 Jul 2026 02:25:27 +0530 Subject: [PATCH 339/364] changed lvlInfo to lvlTalkative --- src/nix/flake.cc | 2 +- src/nix/search.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3fae1f0d6a30..a665e717e1f9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1269,7 +1269,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto attrPathS = attrPath.resolve(*state); - Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", attrPath.to_string(*state))); + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating '%s'", attrPath.to_string(*state))); try { auto recurse = [&]() { diff --git a/src/nix/search.cc b/src/nix/search.cc index d7616bcfb1d0..90646d43a888 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -95,7 +95,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON auto attrPathS = state->symbols.resolve({attrPath}); auto attrPathStr = attrPath.to_string(*state); - Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", attrPathStr)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating '%s'", attrPathStr)); try { auto recurse = [&]() { for (const auto & attr : cursor.getAttrs()) { From 0637027de0e88572b0f2e16141ff6d53920d2ade Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Jul 2026 00:01:00 +0300 Subject: [PATCH 340/364] libutil: Use vfork() once again in runProgram2, but safe(r) now This is a lot. Previously we had used vfork() in a very careless manner - we trampled all over the parent process address space without a care in the world. That used to work ok-ish, but stuff like 9640ab1fa4a88def1c5be31e45cd819caa24a638 shows just how easily broken the status quo is. Some valuable references for this change are: * https://ewontfix.com/7/ - I got some details about vfork() in 9640ab1fa4a88def1c5be31e45cd819caa24a638 wrong. Only the calling thread is suspended - at least on Linux. It's hard to tell what other systems do. At least according to [1] this is a common misconception and actually isn't the case on some other systems?? * Python's migration to vfork https://bugs.python.org/issue35823 * Glibc and musl sources. * https://github.com/python/cpython/blob/main/Modules/_posixsubprocess.c Fixes some bugs along the way: * setuid/setgid and friends reset PDEATHSIG prctl and runProgram2 would set it *before* doing setuid in the child. That could and would lead to orphaned processes as we have seen in 34688ecf5f7af84b969cb72971ef151b63330718. It's still slightly racy, but this change is very large and possibly full of footguns still. * runProgram2 would take the PATH from the environment that the child is supposed to use, not the outer process. This is most certainly a bug that could blow up in our face. Thankfully, it doesn't seem like it was an issue anywhere in the codebase. * With build users, the diff-hook would always fail to restore the mount namespace. This issue was in the general unix runProgram2 which does *restoreProcessContext after* setuid/setgid. It was never surfaced in the old code, because it didn't check that setns actually succeeded....: > [pid 3898066] setns(3, CLONE_NEWNS) = -1 EPERM (Operation not permitted) Which meant that we had the child running in our mount namespace where a readonly store might have been remounted as writable. [1]: https://gist.github.com/nicowilliams/a8a07b0fc75df05f684c23c18d7db234 --- src/libutil/linux/linux-namespaces-private.hh | 11 + src/libutil/linux/linux-namespaces.cc | 8 +- src/libutil/linux/processes.cc | 326 ++++++++++++++++-- src/libutil/unix/signals-private.hh | 12 + src/libutil/unix/signals.cc | 6 +- 5 files changed, 321 insertions(+), 42 deletions(-) create mode 100644 src/libutil/linux/linux-namespaces-private.hh create mode 100644 src/libutil/unix/signals-private.hh diff --git a/src/libutil/linux/linux-namespaces-private.hh b/src/libutil/linux/linux-namespaces-private.hh new file mode 100644 index 000000000000..001a8417901c --- /dev/null +++ b/src/libutil/linux/linux-namespaces-private.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "nix/util/file-descriptor.hh" + +namespace nix { + +extern AutoCloseFD fdSavedMountNamespace; +extern AutoCloseFD fdSavedRoot; +extern bool havePrivateMountNs; + +} // namespace nix diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 816acfba6fcb..6d77d1d58bab 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -3,6 +3,8 @@ #include "nix/util/file-system.hh" #include "nix/util/processes.hh" +#include "linux-namespaces-private.hh" + #include #include @@ -89,9 +91,9 @@ bool mountAndPidNamespacesSupported() ////////////////////////////////////////////////////////////////////// -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -static bool havePrivateMountNs = false; +AutoCloseFD fdSavedMountNamespace; +AutoCloseFD fdSavedRoot; +bool havePrivateMountNs = false; /* Save the current mount namespace so restoreMountNamespace() can return to it later. Ignored if called more than once. */ diff --git a/src/libutil/linux/processes.cc b/src/libutil/linux/processes.cc index cc4687778da8..6b99553372cb 100644 --- a/src/libutil/linux/processes.cc +++ b/src/libutil/linux/processes.cc @@ -4,14 +4,236 @@ #include "nix/util/environment-variables.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" +#include "nix/util/serialise.hh" -#include +#include "linux/linux-namespaces-private.hh" +#include "unix/signals-private.hh" +#include "unix/current-process-private.hh" +#include "util-unix-config-private.hh" + +#include +#include + +#include #include #include +#include +#include +#include #include +#include +#include + +extern char ** environ __attribute__((weak)); namespace nix { +namespace { + +/* This structure intentionally doesn't have any fancy classes from C++ like + std::optional, because I don't trust those. Only plain pointers or integral + types. */ +struct ExecChildParams +{ + const char * program; + const char * chdir; + char * const * environment; + char * const * args; + bool mergeStderrToStdout; + bool lookupPath; + Descriptor stdoutFd; + Descriptor errorPipe; + bool setGid; + gid_t gid; + bool setUid; + uid_t uid; + bool dieWithParent; +}; + +/* + * This code is supposed to run in the child right after vfork(). We never + * return from this function through normal means, but rather always _exit() or + * execv[e] from it. + * + * We shouldn't allow any exceptions to escape and shouldn't allocate any memory, + * or in general do anything that's not async-signal-safe. + * + * There's a slight wrinkle in that in a multithreaded program on Linux, only + * the thread calling vfork() is suspended while other threads of the parent + * continue running. + * + * This also presents a problem with signals, since those will also arrive in + * the child, which shares the address space. Thankfully, we don't have many + * signal handlers that would be negatively affected by this, so we punt on + * the issue. + * + * Another thing worth mentioning is that apparently, WSL and QEMU userspace + * emulation both implement vfork() as plain fork() so the regular footguns of + * fork() apply. + * + * Don't call any functions defined in other translations units from here! + * (for auditability purposes and avoiding accidentally calling something you + * shouldn't). + * + * Even calling libc functions is quite sketchy in general because of lazy + * binding with dynamic linking. Whatever the libc is doing in the dynamic + * linker is likely to not really be safe with vfork(). + * + * Luckily, nixpkgs now builds with eager binding (-z now) so our builds won't + * be affected. TODO: Just build libutil with -z now too, instead of relying on + * hardening defaults. + */ +[[gnu::noinline, noreturn]] static void doExecChild(const ExecChildParams & params) noexcept +{ + auto die = [¶ms] [[noreturn]] (int err, const char * msg) { + [[maybe_unused]] ssize_t ret; /* Swallow all errors. */ + ret = ::write(params.errorPipe, reinterpret_cast(&err), sizeof(err)); + ret = ::write(params.errorPipe, msg, ::strlen(msg)); + ::_exit(1); + }; + + auto dieWithErrno = [&die] [[noreturn]] (const char * msg) { die(errno, msg); }; + + /* TODO: Do something with stdin if we don't want to keep it? */ + + if (params.stdoutFd != INVALID_DESCRIPTOR && ::dup2(params.stdoutFd, STDOUT_FILENO) == -1) + dieWithErrno("dupping stdout"); + + if (params.mergeStderrToStdout && ::dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + dieWithErrno("cannot dup stdout into stderr"); + + if (params.chdir && ::chdir(params.chdir) == -1) + dieWithErrno("chdir failed"); + + /* Restore saved process context. Much like nix::restoreProcessContext, but inlined + and without any possibility of throwing exceptions. */ + + if (havePrivateMountNs) { + char savedCwd[PATH_MAX]; + + /* On Linux, it seems like cwd can't ever be larger than PATH_MAX (as + restricted by the syscall itself). Once again, intentionally not + calling into libc. */ + if (::syscall(SYS_getcwd, savedCwd, sizeof(savedCwd)) == -1) + dieWithErrno("getcwd failed"); + + if (::setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + dieWithErrno("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (::fchdir(fdSavedRoot.get())) + dieWithErrno("chdir into saved root"); + + if (::chroot(".")) + dieWithErrno("chroot into saved root"); + } + + if (::chdir(savedCwd) == -1) + dieWithErrno("restoring cwd"); + } + + /* Important! Calling syscalls directly and not libc functions because of a + discrepancy in POSIX specification (i.e. POSIX setgid/setuid has to apply + to all threads, while the syscall only applies to a task). + A huge footgun is apparently the fact that glibc defines SYS_setgid to the 16 bit + version of the syscall, while musl "fixes it up" to use the 32 bit version. + The end result is that we have to make sure to use the right one still. */ + +#ifdef SYS_setuid32 +# define NIX_SYS_setuid SYS_setuid32 +# define NIX_SYS_setgid SYS_setgid32 +# define NIX_SYS_setgroups SYS_setgroups32 +#else +# define NIX_SYS_setuid SYS_setuid +# define NIX_SYS_setgid SYS_setgid +# define NIX_SYS_setgroups SYS_setgroups +#endif + + if (params.setGid && ::syscall(NIX_SYS_setgid, params.gid) == -1) + dieWithErrno("setgid failed"); + + /* Drop all other groups if we're setgid. */ + if (params.setGid && ::syscall(NIX_SYS_setgroups, 0, 0) == -1) + dieWithErrno("setgroups failed"); + + if (params.setUid && ::syscall(NIX_SYS_setuid, params.uid) == -1) + dieWithErrno("setuid failed"); + + /* Technically slightly racy, we might want to do something like what + preserveDeathSignal does. */ + if (params.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + dieWithErrno("setting death signal"); + +#undef NIX_SYS_setuid +#undef NIX_SYS_setgid +#undef NIX_SYS_setgroups + + if (unix::savedStackSize) { + struct ::rlimit limit; + if (::getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = unix::savedStackSize; + ::setrlimit(RLIMIT_STACK, &limit); + /* TODO: Why do we ignore all errors here? */ + } + } + + /* Like unix::restoreSignals(), but safe to do in a vfork child. */ + if (unix::savedSignalMaskIsSet && sigprocmask(SIG_SETMASK, &unix::savedSignalMask, nullptr) == -1) + dieWithErrno("restoring signals"); + + /* TODO: Close leaked file descriptors? The best way is with close_range(). */ + + if (params.lookupPath) + /* Nonstandard, but both musl and glibc have it and it doesn't + seem to do anything weird or allocate memory, so it should + be fine-ish? The use of execvp has some footguns though (see + https://github.com/NixOS/nix/pull/9494) and maybe we should get rid + of it entirely and do executable path resolution in the parent. + The hacky ENOEXEC handling also doesn't seem to exist in musl. + + This does path lookup in the global environment of the parent, and + not in the params.environment like it's done in runProgram2 for other + unixes (arguably a bugfix)! + + Don't accidentally call nix::execvpe! */ + ::execvpe(params.program, params.args, params.environment); + else + ::execve(params.program, params.args, params.environment); + + dieWithErrno("could not exec program"); +} + +[[gnu::noinline]] static pid_t startExecChildInVFork(const ExecChildParams & params) noexcept +{ + pid_t pid = ::vfork(); + + /* In the unlikely scenario that vfork() fails we return -1 here too. */ + if (pid != 0) + return pid; + + /* Now run the actual child. */ + doExecChild(params); +} + +/* TODO: This can be factored out if this becomes useful for other platforms? */ +static Strings prepareEnvironmentStrings(const StringMap & environment) +{ + Strings env; + for (auto & [name, value] : environment) { + std::string var; + var.reserve(name.size() + value.size() + 1); + var += name; + var += "="; + var += value; + env.push_back(std::move(var)); + } + return env; +} + +} // namespace + +/* TODO: Factor this out into a `launchProgram` that returns a pid. That would be + much more useful in more places. */ void runProgram2(const RunOptions & options) { checkInterrupt(); @@ -21,48 +243,78 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); - ProcessOptions processOptions; + /* Pipe that the child reports errors through. */ + Pipe childErrorPipe; + childErrorPipe.create(); + + /* Prepare arguments and environment for the child. */ + Strings args_(options.args); + args_.push_front(options.program.native()); + const Strings env_ = options.environment ? prepareEnvironmentStrings(*options.environment) : Strings{}; + const auto env = stringsToCharPtrs(env_); + const auto args = stringsToCharPtrs(args_); + + const ExecChildParams params = { + .program = options.program.c_str(), + .chdir = options.chdir ? options.chdir->c_str() : nullptr, + .environment = options.environment ? env.data() : environ, + .args = args.data(), + .mergeStderrToStdout = options.mergeStderrToStdout, + .lookupPath = options.lookupPath, + .stdoutFd = options.standardOut ? out.writeSide.get() : INVALID_DESCRIPTOR, + .errorPipe = childErrorPipe.writeSide.get(), + .setGid = options.gid.has_value(), + /* The default is not used, but a bit sketchy to leave zero initialised so "nobody". */ + .gid = options.gid.value_or(65534), + .setUid = options.uid.has_value(), + /* The default is not used, but a bit sketchy to leave zero initialised so "nobody". */ + .uid = options.uid.value_or(65534), + .dieWithParent = true, /* TODO: Maybe we might want to expose this in RunOptions? */ + }; auto suspension = logger->suspendIf(options.isInteractive); + const auto savedErrno = errno; + /* Fork. */ - Pid pid = startProcess( - [&] { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program.native()); - - restoreProcessContext(); - - if (options.lookupPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing %s", PathFmt(options.program)); - }, - processOptions); + Pid pid = startExecChildInVFork(params); + + /* errno is also shared with the child, so it can trample it. Restoring it + doesn't necessarily matter much though. */ + const auto forkErrno = errno; + errno = savedErrno; + + if (pid == -1) + throw SysError(forkErrno, "unable to vfork"); out.writeSide.close(); + childErrorPipe.writeSide.close(); + + StringSink childErrorSink; + drainFD(childErrorPipe.readSide.get(), childErrorSink); + + /* We don't write anything to the pipe on success. */ + if (const auto & errorContent = childErrorSink.s; errorContent.size()) { + int status = pid.wait(); + + auto execErr = ExecError(status, "could not start program %1%", PathFmt(options.program)); + + /* The child returned garbage through the error pipe. I think it's + pretty unlikely that this will happen, but maybe a signal arriving + could result in ::write failing with EINTR. It's unclear to me + whether that that can happen with writes under PIPE_BUF? */ + if (errorContent.size() >= sizeof(int)) { + auto intBytes = errorContent.substr(0, sizeof(int)); + int errNo; + std::memcpy(&errNo, intBytes.data(), sizeof(errNo)); + auto errMsg = errorContent.substr(sizeof(int)); + /* It's a bit strange, but the interface of runProgram2 reports + spawn helper errors as ExecError. */ + execErr.addTrace({}, HintFmt("spawn helper process failed: %1%: %2%", errMsg, ::strerror(errNo))); + } + + throw std::move(execErr); + } if (options.standardOut) drainFD(out.readSide.get(), *options.standardOut); diff --git a/src/libutil/unix/signals-private.hh b/src/libutil/unix/signals-private.hh new file mode 100644 index 000000000000..ef2b5cc16f8e --- /dev/null +++ b/src/libutil/unix/signals-private.hh @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace nix { +namespace unix { + +extern sigset_t savedSignalMask; +extern bool savedSignalMaskIsSet; + +} // namespace unix +} // namespace nix diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index 38030a77ddfd..3a63276fc771 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -5,6 +5,8 @@ #include "nix/util/sync.hh" #include "nix/util/terminal.hh" +#include "unix/signals-private.hh" + #include namespace nix { @@ -101,8 +103,8 @@ void unix::triggerInterrupt() } } -static sigset_t savedSignalMask; -static bool savedSignalMaskIsSet = false; +sigset_t unix::savedSignalMask; +bool unix::savedSignalMaskIsSet = false; void unix::saveSignalMask() { From f70b87e17cf5f4fe2628c2180f1b7b515ac30d60 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Jul 2026 00:29:00 +0300 Subject: [PATCH 341/364] clang-tidy: Add lint for using namespace --- .../common/clang-tidy/.clang-tidy | 2 +- src/clang-tidy-plugin/meson.build | 1 + .../nix-clang-tidy-checks.cc | 15 ++++---- src/clang-tidy-plugin/nix-using-namespace.cc | 26 ++++++++++++++ src/clang-tidy-plugin/nix-using-namespace.hh | 36 +++++++++++++++++++ 5 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 src/clang-tidy-plugin/nix-using-namespace.cc create mode 100644 src/clang-tidy-plugin/nix-using-namespace.hh diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index b0cc92429c4e..24f02a1fd943 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -80,7 +80,7 @@ Checks: - cppcoreguidelines-missing-std-forward - android-cloexec-open - android-cloexec-pipe2 - # Custom nix checks (when added) + # Custom nix checks - nix-* CheckOptions: diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build index 7896bd085479..150e8aafcc3e 100644 --- a/src/clang-tidy-plugin/meson.build +++ b/src/clang-tidy-plugin/meson.build @@ -20,6 +20,7 @@ llvm_dep = dependency('LLVM', version : '>= 16', required : true) sources = files( 'nix-clang-tidy-checks.cc', + 'nix-using-namespace.cc', ) # Build as a shared module (plugin) that can be loaded by clang-tidy --load diff --git a/src/clang-tidy-plugin/nix-clang-tidy-checks.cc b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc index 64ec63c2d319..bf69e7f1154f 100644 --- a/src/clang-tidy-plugin/nix-clang-tidy-checks.cc +++ b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc @@ -13,22 +13,19 @@ #include #include -namespace nix::clang_tidy { +#include "nix-using-namespace.hh" -using namespace clang; -using namespace clang::tidy; +namespace nix::clang_tidy { -class NixClangTidyChecks : public ClangTidyModule +class NixClangTidyChecks : public clang::tidy::ClangTidyModule { public: - void addCheckFactories([[maybe_unused]] ClangTidyCheckFactories & CheckFactories) override + void addCheckFactories([[maybe_unused]] clang::tidy::ClangTidyCheckFactories & CheckFactories) override { - // Custom checks will be registered here. - // Example: - // CheckFactories.registerCheck("nix-my-custom-check"); + CheckFactories.registerCheck("nix-using-namespace"); } }; -static ClangTidyModuleRegistry::Add X("nix-module", "Adds Nix-specific checks"); +static clang::tidy::ClangTidyModuleRegistry::Add X("nix-module", "Adds Nix-specific checks"); } // namespace nix::clang_tidy diff --git a/src/clang-tidy-plugin/nix-using-namespace.cc b/src/clang-tidy-plugin/nix-using-namespace.cc new file mode 100644 index 000000000000..69e1fdf73a84 --- /dev/null +++ b/src/clang-tidy-plugin/nix-using-namespace.cc @@ -0,0 +1,26 @@ +#include "nix-using-namespace.hh" + +#include +#include + +namespace nix::clang_tidy { + +void UsingNamespaceInNamespaceScopeCheck::registerMatchers(clang::ast_matchers::MatchFinder * Finder) +{ + Finder->addMatcher(clang::ast_matchers::usingDirectiveDecl().bind("usingNamespace"), this); +} + +void UsingNamespaceInNamespaceScopeCheck::check(const clang::ast_matchers::MatchFinder::MatchResult & Result) +{ + const auto * U = Result.Nodes.getNodeAs("usingNamespace"); + const clang::SourceLocation Loc = U->getBeginLoc(); + if (U->isImplicit() || !Loc.isValid() || U->getParentFunctionOrMethod()) + return; + + diag( + Loc, + "do not use using namespace directive in namespace scopes - keep those local to functions or explicitly qualify names." + "This is to reduce namespace pollution with unity builds."); +} + +} // namespace nix::clang_tidy diff --git a/src/clang-tidy-plugin/nix-using-namespace.hh b/src/clang-tidy-plugin/nix-using-namespace.hh new file mode 100644 index 000000000000..48cc3ebcb5f2 --- /dev/null +++ b/src/clang-tidy-plugin/nix-using-namespace.hh @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace nix::clang_tidy { + +/** + * Check that forbids instances on `using namespace ...;` in a namespace + * scope. + * + * This is because we rely on unity builds in certain situations (faster + * non-incremental builds, static initialiser issues), and the common pattern of + * doing `using namespace` in a translation unit is a big footgun. + * + * Based on `google-build-using-namespace`, modulo that `using namespace` in a + * non-namespace scope is fine (like in a function). + */ +class UsingNamespaceInNamespaceScopeCheck : public clang::tidy::ClangTidyCheck +{ +public: + UsingNamespaceInNamespaceScopeCheck(llvm::StringRef Name, clang::tidy::ClangTidyContext * Context) + : ClangTidyCheck(Name, Context) + { + } + + bool isLanguageVersionSupported(const clang::LangOptions & LangOpts) const override + { + return LangOpts.CPlusPlus; + } + + void registerMatchers(clang::ast_matchers::MatchFinder * Finder) override; + + void check(const clang::ast_matchers::MatchFinder::MatchResult & Result) override; +}; + +} // namespace nix::clang_tidy From 5bc46d5efe76f2cfac920dd400fb9c79143150b1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Jul 2026 02:36:31 +0300 Subject: [PATCH 342/364] Fix using namespace instances in namespace scoped found by nix-using-namespace lint In a60d54c1a53eafd93756b2f1e24338fff02a5922 I introduced a bit of tech debt around the fact that with unity builds it's very ill-advised to use namespace-level using namespace directives (due to the simple fact that they leak into every other translation unit that's merged lexically). The previous commit introduced a custom linter to diagnose problematic instances of this pattern and issues identified by it are fixed now. This is a bit annoying, but it doesn't seem to be that high of a cost considering we get so much out of unity builds (i.e. yet to be merged static builds on darwin with static initialisers, much faster compilation etc.). The linter explains the reasoning pretty well, so the ongoing maintenance is going to be pretty minimal. See https://github.com/NixOS/nix/pull/15656 for a previous whack-a-mole style removal of top-level using namespace directives. --- .../tests/value/context.cc | 7 ++- src/libexpr-tests/error_traces.cc | 4 +- src/libexpr-tests/value/print.cc | 2 - src/libfetchers/fetchers.cc | 7 +-- src/libfetchers/git.cc | 2 - .../include/nix/fetchers/fetchers.hh | 2 +- src/libfetchers/mercurial.cc | 4 +- src/libstore-test-support/derived-path.cc | 16 +++-- src/libstore-test-support/path.cc | 2 +- src/libstore-tests/build-result.cc | 6 +- .../derivation-advanced-attrs.cc | 5 +- src/libstore-tests/http-binary-cache-store.cc | 9 +-- src/libstore/build-result.cc | 17 ++++-- src/libstore/build/derivation-builder.cc | 7 +-- src/libstore/content-address.cc | 13 ++-- src/libstore/derivation-options.cc | 56 ++++++++++-------- src/libstore/derivations.cc | 56 ++++++++++++------ src/libstore/derived-path.cc | 39 ++++++------ src/libstore/downstream-placeholder.cc | 14 +++-- src/libstore/dummy-store.cc | 19 +++--- .../include/nix/store/outputs-spec.hh | 4 +- .../include/nix/store/store-reference.hh | 2 +- src/libstore/misc.cc | 7 +-- src/libstore/nar-info.cc | 10 ++-- src/libstore/outputs-spec.cc | 16 ++--- src/libstore/path-info.cc | 22 +++---- src/libstore/path.cc | 8 +-- src/libstore/realisation.cc | 19 +++--- src/libstore/s3-url.cc | 4 +- src/libstore/store-reference.cc | 8 +-- src/libstore/unix/build/hook-instance.cc | 4 +- src/libutil-test-support/hash.cc | 5 +- .../include/nix/util/tests/hash.hh | 6 +- src/libutil-tests/checked-arithmetic.cc | 2 - src/libutil-tests/closure.cc | 14 ++--- src/libutil-tests/file-system.cc | 4 +- src/libutil-tests/memory-source-accessor.cc | 2 +- src/libutil/base-n.cc | 2 - src/libutil/git.cc | 5 +- src/libutil/hash.cc | 12 ++-- src/libutil/include/nix/util/hash.hh | 2 +- src/libutil/include/nix/util/json-impls.hh | 21 ++++--- .../nix/util/memory-source-accessor.hh | 14 ++--- src/libutil/memory-source-accessor/json.cc | 59 ++++++++++--------- src/libutil/signature/local-keys.cc | 7 ++- src/libutil/unix/processes.cc | 3 +- src/nix/nix-build/nix-build.cc | 5 +- src/nix/nix-store/nix-store.cc | 2 +- 48 files changed, 292 insertions(+), 264 deletions(-) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index 22f8aa7cf0ff..2483b3ea811b 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -4,10 +4,10 @@ #include "nix/expr/tests/value/context.hh" namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::map(gen::arbitrary(), [](StorePath drvPath) { return NixStringContextElem::DrvDeep{ .drvPath = drvPath, @@ -15,8 +15,9 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat( gen::inRange(0, std::variant_size_v), [](uint8_t n) -> Gen { diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index 9f2d1f92fa3a..918ea71807e2 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -5,14 +5,14 @@ namespace nix { -using namespace testing; - // Testing eval of PrimOp's class ErrorTraceTest : public LibExprTest {}; TEST_F(ErrorTraceTest, TraceBuilder) { + using namespace testing; + ASSERT_THROW(state.error("puppy").debugThrow(), EvalError); ASSERT_THROW(state.error("puppy").withTrace(noPos, "doggy").debugThrow(), EvalError); diff --git a/src/libexpr-tests/value/print.cc b/src/libexpr-tests/value/print.cc index 654a50b0ae0b..0082b6eac75e 100644 --- a/src/libexpr-tests/value/print.cc +++ b/src/libexpr-tests/value/print.cc @@ -6,8 +6,6 @@ namespace nix { -using namespace testing; - struct ValuePrintingTests : LibExprTest { template diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6d7266e09dbb..53bcf12dbe69 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -525,12 +525,11 @@ std::string publicKeys_to_string(const std::vector & publicKeys) namespace nlohmann { -using namespace nix; - #ifndef DOXYGEN_SKIP -fetchers::PublicKey adl_serializer::from_json(const json & json) +nix::fetchers::PublicKey adl_serializer::from_json(const json & json) { + using namespace nix; fetchers::PublicKey res = {}; auto & obj = getObject(json); if (auto * type = optionalValueAt(obj, "type")) @@ -541,7 +540,7 @@ fetchers::PublicKey adl_serializer::from_json(const json & return res; } -void adl_serializer::to_json(json & json, const fetchers::PublicKey & p) +void adl_serializer::to_json(json & json, const nix::fetchers::PublicKey & p) { json["type"] = p.type; json["key"] = p.key; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 29fcb5dfc352..f0d19945be10 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,8 +22,6 @@ # include #endif -using namespace std::string_literals; - namespace nix::fetchers { namespace { diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 180d10e9dbbf..f65bbac73d4f 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -296,4 +296,4 @@ std::string publicKeys_to_string(const std::vector &); } // namespace nix::fetchers -JSON_IMPL(fetchers::PublicKey) +JSON_IMPL(nix::fetchers::PublicKey) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index ab1d31ed330f..6a4d239a03cf 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -14,8 +14,6 @@ #include #include -using namespace std::string_literals; - namespace nix::fetchers { static RunOptions hgOptions(OsStrings args) @@ -227,6 +225,8 @@ struct MercurialInputScheme : InputScheme input.attrs.insert_or_assign("ref", chomp(runHg({OS_STR("branch"), OS_STR("-R"), localPath.native()}))); + using namespace std::string_literals; + auto files = tokenizeString( runHg({ OS_STR("status"), diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index c27edc95ef9b..8440a74df912 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -4,10 +4,10 @@ #include "nix/store/tests/derived-path.hh" namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::map(gen::arbitrary(), [](StorePath path) { return DerivedPath::Opaque{ .path = path, @@ -15,8 +15,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::arbitrary(), [](SingleDerivedPath drvPath) { return gen::map(gen::arbitrary(), [drvPath](StorePathName outputPath) { return SingleDerivedPath::Built{ @@ -27,8 +28,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::arbitrary(), [](SingleDerivedPath drvPath) { return gen::map(gen::arbitrary(), [drvPath](OutputsSpec outputs) { return DerivedPath::Built{ @@ -39,8 +41,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::inRange(0, std::variant_size_v), [](uint8_t n) { switch (n) { case 0: @@ -53,8 +56,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::inRange(0, std::variant_size_v), [](uint8_t n) { switch (n) { case 0: diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index bca404cde455..ce14f469c7f1 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -55,7 +55,7 @@ Gen Arbitrary::arbitrary() })); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { return gen::construct( gen::arbitrary(), diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index a1d8ddee6412..9215b99e041f 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -36,8 +36,6 @@ TEST_P(BuildResultJsonTest, to_json) writeJsonTest(name, value); } -using namespace std::literals::chrono_literals; - INSTANTIATE_TEST_SUITE_P( BuildResultJSON, BuildResultJsonTest, @@ -89,8 +87,8 @@ INSTANTIATE_TEST_SUITE_P( .timesBuilt = 3, .startTime = 30, .stopTime = 50, - .cpuUser = std::chrono::microseconds(500s), - .cpuSystem = std::chrono::microseconds(604s), + .cpuUser = std::chrono::seconds(500), + .cpuSystem = std::chrono::seconds(604), }, })); diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index bb7ad28f3d78..6d2c053c0d65 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -12,8 +12,6 @@ namespace nix { -using namespace nlohmann; - class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, public LibStoreTest { protected: @@ -83,6 +81,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); #define TEST_ATERM_JSON(STEM, NAME) \ TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ { \ + using namespace nlohmann; \ this->readTest(NAME ".json", [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ /* Use DRV file instead of C++ literal as source of truth. */ \ @@ -95,6 +94,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); \ TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ { \ + using namespace nlohmann; \ this->writeTest( \ NAME ".json", \ [&]() -> json { \ @@ -108,6 +108,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); \ TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ { \ + using namespace nlohmann; \ this->readTest(NAME ".drv", [&](auto encoded) { \ /* Use JSON file instead of C++ literal as source of truth. */ \ auto j = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index 74f3b93cd3df..93d279d53a1f 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -57,13 +57,10 @@ TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams) using testing::HttpsBinaryCacheStoreMtlsTest; using testing::HttpsBinaryCacheStoreTest; -using namespace std::string_view_literals; -using namespace std::string_literals; - TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo) { auto store = openStore(makeConfig()); - StringSource dump{"test"sv}; + StringSource dump{std::string_view("test")}; auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); EXPECT_NO_THROW(store->queryPathInfo(path)); } @@ -74,7 +71,7 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo) config->tlsCert = clientCert; config->tlsKey = clientKey; auto store = openStore(config); - StringSource dump{"test"sv}; + StringSource dump{std::string_view("test")}; auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); EXPECT_NO_THROW(store->queryPathInfo(path)); } @@ -104,7 +101,7 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert) TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority) { - StringSource dump{"test"sv}; + StringSource dump{std::string_view("test")}; auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); for (auto & entry : DirectoryIterator{cacheDir}) diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index c9d78e5ccf30..7dd6954f607c 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -145,10 +145,10 @@ std::strong_ordering BuildError::operator<=>(const BuildError & other) const noe namespace nlohmann { -using namespace nix; - -void adl_serializer::to_json(json & res, const BuildResult & br) +void adl_serializer::to_json(json & res, const nix::BuildResult & br) { + using namespace nix; + res = json::object(); // Common fields @@ -181,8 +181,10 @@ void adl_serializer::to_json(json & res, const BuildResult & br) br.inner); } -BuildResult adl_serializer::from_json(const json & _json) +nix::BuildResult adl_serializer::from_json(const json & _json) { + using namespace nix; + auto & json = getObject(_json); BuildResult br; @@ -219,8 +221,10 @@ BuildResult adl_serializer::from_json(const json & _json) return br; } -KeyedBuildResult adl_serializer::from_json(const json & json0) +nix::KeyedBuildResult adl_serializer::from_json(const json & json0) { + using namespace nix; + auto json = getObject(json0); return KeyedBuildResult{ @@ -229,8 +233,9 @@ KeyedBuildResult adl_serializer::from_json(const json & json0) }; } -void adl_serializer::to_json(json & json, const KeyedBuildResult & kbr) +void adl_serializer::to_json(json & json, const nix::KeyedBuildResult & kbr) { + using namespace nix; adl_serializer::to_json(json, kbr); json["path"] = kbr.path; } diff --git a/src/libstore/build/derivation-builder.cc b/src/libstore/build/derivation-builder.cc index 73404a08dcb8..a38f7b2bc029 100644 --- a/src/libstore/build/derivation-builder.cc +++ b/src/libstore/build/derivation-builder.cc @@ -3,10 +3,9 @@ namespace nlohmann { -using namespace nix; - -ExternalBuilder adl_serializer::from_json(const json & json) +nix::ExternalBuilder adl_serializer::from_json(const json & json) { + using namespace nix; auto obj = getObject(json); return { .systems = valueAt(obj, "systems"), @@ -15,7 +14,7 @@ ExternalBuilder adl_serializer::from_json(const json & json) }; } -void adl_serializer::to_json(json & json, const ExternalBuilder & eb) +void adl_serializer::to_json(json & json, const nix::ExternalBuilder & eb) { json = { {"systems", eb.systems}, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 497c2c5b47c1..2c40e6f6dd31 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -304,20 +304,19 @@ Hash ContentAddressWithReferences::getHash() const namespace nlohmann { -using namespace nix; - -ContentAddressMethod adl_serializer::from_json(const json & json) +nix::ContentAddressMethod adl_serializer::from_json(const json & json) { - return ContentAddressMethod::parse(getString(json)); + return nix::ContentAddressMethod::parse(nix::getString(json)); } -void adl_serializer::to_json(json & json, const ContentAddressMethod & m) +void adl_serializer::to_json(json & json, const nix::ContentAddressMethod & m) { json = m.render(); } -ContentAddress adl_serializer::from_json(const json & json) +nix::ContentAddress adl_serializer::from_json(const json & json) { + using namespace nix; auto obj = getObject(json); return { .method = adl_serializer::from_json(valueAt(obj, "method")), @@ -325,7 +324,7 @@ ContentAddress adl_serializer::from_json(const json & json) }; } -void adl_serializer::to_json(json & json, const ContentAddress & ca) +void adl_serializer::to_json(json & json, const nix::ContentAddress & ca) { json = { {"method", ca.method}, diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index b3d4261b5464..ab73e921fcea 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -536,11 +536,11 @@ template struct DerivationOptions; namespace nlohmann { -using namespace nix; - template -static DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) +static nix::DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) { + using namespace nix; + auto & json = getObject(json_); return { @@ -576,8 +576,10 @@ static DerivationOptions derivationOptionsFromJson(const nlohmann::json } template -static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptions & o) +static void derivationOptionsToJson(nlohmann::json & json, const nix::DerivationOptions & o) { + using namespace nix; + json["outputChecks"] = std::visit( overloaded{ [&](const OutputChecks & checks) { @@ -609,8 +611,10 @@ static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptio } template -static OutputChecks outputChecksFromJson(const nlohmann::json & json_) +static nix::OutputChecks outputChecksFromJson(const nlohmann::json & json_) { + using namespace nix; + auto & json = getObject(json_); return { @@ -625,7 +629,7 @@ static OutputChecks outputChecksFromJson(const nlohmann::json & json_) } template -static void outputChecksToJson(nlohmann::json & json, const OutputChecks & c) +static void outputChecksToJson(nlohmann::json & json, const nix::OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["maxSize"] = c.maxSize; @@ -636,45 +640,51 @@ static void outputChecksToJson(nlohmann::json & json, const OutputChecks json["disallowedRequisites"] = c.disallowedRequisites; } -DerivationOptions adl_serializer>::from_json(const json & json_) +nix::DerivationOptions +adl_serializer>::from_json(const json & json_) { - return derivationOptionsFromJson(json_); + return derivationOptionsFromJson(json_); } -void adl_serializer>::to_json( - json & json, const DerivationOptions & o) +void adl_serializer>::to_json( + json & json, const nix::DerivationOptions & o) { - derivationOptionsToJson(json, o); + derivationOptionsToJson(json, o); } -DerivationOptions adl_serializer>::from_json(const json & json_) +nix::DerivationOptions +adl_serializer>::from_json(const json & json_) { - return derivationOptionsFromJson(json_); + return derivationOptionsFromJson(json_); } -void adl_serializer>::to_json(json & json, const DerivationOptions & o) +void adl_serializer>::to_json( + json & json, const nix::DerivationOptions & o) { - derivationOptionsToJson(json, o); + derivationOptionsToJson(json, o); } -OutputChecks adl_serializer>::from_json(const json & json_) +nix::OutputChecks +adl_serializer>::from_json(const json & json_) { - return outputChecksFromJson(json_); + return outputChecksFromJson(json_); } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +void adl_serializer>::to_json( + json & json, const nix::OutputChecks & c) { - outputChecksToJson(json, c); + outputChecksToJson(json, c); } -OutputChecks adl_serializer>::from_json(const json & json_) +nix::OutputChecks adl_serializer>::from_json(const json & json_) { - return outputChecksFromJson(json_); + return outputChecksFromJson(json_); } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +void adl_serializer>::to_json( + json & json, const nix::OutputChecks & c) { - outputChecksToJson(json, c); + outputChecksToJson(json, c); } } // namespace nlohmann diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0cf79e82f75f..d25dc19cd493 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,8 +15,6 @@ namespace nix { -using namespace std::literals::string_view_literals; - BasicDerivation::~BasicDerivation() {} Derivation::~Derivation() {} @@ -316,6 +314,8 @@ static DerivationOutput parseDerivationOutput( std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { + using namespace std::literals::string_view_literals; + if (!hashAlgoStr.empty()) { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); if (method == ContentAddressMethod::Raw::Text) @@ -395,6 +395,8 @@ enum struct DerivationATermVersion { static DerivedPathMap::ChildNode parseDerivedPathMapNode(const StoreDirConfig & store, StringViewStream & str, DerivationATermVersion version) { + using namespace std::literals::string_view_literals; + DerivedPathMap::ChildNode node; auto parseNonDynamic = [&]() { node.value = parseStrings(str, false); }; @@ -440,6 +442,8 @@ Derivation parseDerivation( std::string_view name, const ExperimentalFeatureSettings & xpSettings) { + using namespace std::literals::string_view_literals; + Derivation drv; drv.name = name; @@ -600,6 +604,8 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt static void unparseDerivedPathMapNode( const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) { + using namespace std::literals::string_view_literals; + s += ','; if (node.childMap.empty()) { printUnquotedStrings(s, node.value.begin(), node.value.end()); @@ -643,6 +649,8 @@ static bool hasDynamicDrvDep(const Derivation & drv) std::string Derivation::unparse( const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { + using namespace std::literals::string_view_literals; + std::string s; s.reserve(65536); @@ -790,6 +798,8 @@ bool isDerivation(std::string_view fileName) std::string outputPathName(std::string_view drvName, OutputNameView outputName) { + using namespace std::literals::string_view_literals; + std::string res{drvName}; if (outputName != "out"sv) { res += '-'; @@ -800,6 +810,8 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName) DerivationType BasicDerivation::type() const { + using namespace std::literals::string_view_literals; + std::optional floatingHashAlgo; std::optional ty; @@ -1435,10 +1447,9 @@ const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); namespace nlohmann { -using namespace nix; - -void adl_serializer::to_json(json & res, const DerivationOutput & o) +void adl_serializer::to_json(json & res, const nix::DerivationOutput & o) { + using namespace nix; res = nlohmann::json::object(); std::visit( overloaded{ @@ -1466,9 +1477,10 @@ void adl_serializer::to_json(json & res, const DerivationOutpu o.raw); } -DerivationOutput -adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +nix::DerivationOutput adl_serializer::from_json( + const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; std::set keys; auto & json = getObject(_json); @@ -1532,15 +1544,16 @@ adl_serializer::from_json(const json & _json, const Experiment } } -static void inputSrcsToJson(json & res, const StorePathSet & inputSrcs) +static void inputSrcsToJson(json & res, const nix::StorePathSet & inputSrcs) { res = nlohmann::json::array(); for (auto & input : inputSrcs) res.emplace_back(input); } -static void basicDerivationToJson(json & res, const BasicDerivation & d) +static void basicDerivationToJson(json & res, const nix::BasicDerivation & d) { + using namespace nix; res = nlohmann::json::object(); res["name"] = d.name; @@ -1562,15 +1575,17 @@ static void basicDerivationToJson(json & res, const BasicDerivation & d) res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -void adl_serializer::to_json(json & res, const BasicDerivation & d) +void adl_serializer::to_json(json & res, const nix::BasicDerivation & d) { basicDerivationToJson(res, d); inputSrcsToJson(res["inputs"], d.inputSrcs); } -void adl_serializer::to_json(json & res, const Derivation & d) +void adl_serializer::to_json(json & res, const nix::Derivation & d) { + using namespace nix; + basicDerivationToJson(res, d); { @@ -1598,16 +1613,18 @@ void adl_serializer::to_json(json & res, const Derivation & d) } } -static void inputSrcsFromJson(const json & inputSrcsJson, StorePathSet & inputSrcs) +static void inputSrcsFromJson(const json & inputSrcsJson, nix::StorePathSet & inputSrcs) { - auto arr = getArray(inputSrcsJson); + auto arr = nix::getArray(inputSrcsJson); for (auto & input : arr) inputSrcs.insert(input); } static void basicDerivationFromJson( - const json::object_t & json, BasicDerivation & res, const ExperimentalFeatureSettings & xpSettings) + const json::object_t & json, nix::BasicDerivation & res, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; + res.name = getString(valueAt(json, "name")); { @@ -1645,9 +1662,11 @@ static void basicDerivationFromJson( res.structuredAttrs = StructuredAttrs{*structuredAttrs}; } -BasicDerivation -adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +nix::BasicDerivation +adl_serializer::from_json(const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; + BasicDerivation res; auto & json = getObject(_json); basicDerivationFromJson(json, res, xpSettings); @@ -1662,8 +1681,11 @@ adl_serializer::from_json(const json & _json, const Experimenta return res; } -Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +nix::Derivation +adl_serializer::from_json(const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; + Derivation res; auto & json = getObject(_json); basicDerivationFromJson(json, res, xpSettings); diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 131674aa5595..0aa7c4d34c35 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -224,17 +224,17 @@ const StorePath & DerivedPath::getBaseStorePath() const namespace nlohmann { -void adl_serializer::to_json(json & json, const SingleDerivedPath::Opaque & o) +void adl_serializer::to_json(json & json, const nix::SingleDerivedPath::Opaque & o) { json = o.path; } -SingleDerivedPath::Opaque adl_serializer::from_json(const json & json) +nix::SingleDerivedPath::Opaque adl_serializer::from_json(const json & json) { - return SingleDerivedPath::Opaque{json}; + return {json}; } -void adl_serializer::to_json(json & json, const SingleDerivedPath::Built & sdpb) +void adl_serializer::to_json(json & json, const nix::SingleDerivedPath::Built & sdpb) { json = { {"drvPath", *sdpb.drvPath}, @@ -242,7 +242,7 @@ void adl_serializer::to_json(json & json, const Single }; } -void adl_serializer::to_json(json & json, const DerivedPath::Built & dbp) +void adl_serializer::to_json(json & json, const nix::DerivedPath::Built & dbp) { json = { {"drvPath", *dbp.drvPath}, @@ -250,9 +250,10 @@ void adl_serializer::to_json(json & json, const DerivedPath: }; } -SingleDerivedPath::Built -adl_serializer::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings) +nix::SingleDerivedPath::Built adl_serializer::from_json( + const json & json0, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; auto & json = getObject(json0); auto drvPath = make_ref(static_cast(valueAt(json, "drvPath"))); drvRequireExperiment(*drvPath, xpSettings); @@ -262,9 +263,10 @@ adl_serializer::from_json(const json & json0, const Ex }; } -DerivedPath::Built -adl_serializer::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings) +nix::DerivedPath::Built adl_serializer::from_json( + const json & json0, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; auto & json = getObject(json0); auto drvPath = make_ref(static_cast(valueAt(json, "drvPath"))); drvRequireExperiment(*drvPath, xpSettings); @@ -274,31 +276,32 @@ adl_serializer::from_json(const json & json0, const Experime }; } -void adl_serializer::to_json(json & json, const SingleDerivedPath & sdp) +void adl_serializer::to_json(json & json, const nix::SingleDerivedPath & sdp) { std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); } -void adl_serializer::to_json(json & json, const DerivedPath & sdp) +void adl_serializer::to_json(json & json, const nix::DerivedPath & sdp) { std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); } -SingleDerivedPath -adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) +nix::SingleDerivedPath adl_serializer::from_json( + const json & json, const nix::ExperimentalFeatureSettings & xpSettings) { if (json.is_string()) - return static_cast(json); + return static_cast(json); else - return adl_serializer::from_json(json, xpSettings); + return adl_serializer::from_json(json, xpSettings); } -DerivedPath adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) +nix::DerivedPath +adl_serializer::from_json(const json & json, const nix::ExperimentalFeatureSettings & xpSettings) { if (json.is_string()) - return static_cast(json); + return static_cast(json); else - return adl_serializer::from_json(json, xpSettings); + return adl_serializer::from_json(json, xpSettings); } } // namespace nlohmann diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 73ed2b74a7b6..4a73e9daf3ed 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -53,11 +53,11 @@ DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( namespace nlohmann { -using namespace nix; - template -DrvRef adl_serializer>::from_json(const json & json) +nix::DrvRef adl_serializer>::from_json(const json & json) { + using namespace nix; + // OutputName case: { "drvPath": "self", "output": } if (json.type() == nlohmann::json::value_t::object) { auto & obj = getObject(json); @@ -74,8 +74,10 @@ DrvRef adl_serializer>::from_json(const json & json) } template -void adl_serializer>::to_json(json & json, const DrvRef & ref) +void adl_serializer>::to_json(json & json, const nix::DrvRef & ref) { + using namespace nix; + std::visit( overloaded{ [&](const OutputName & outputName) { @@ -88,7 +90,7 @@ void adl_serializer>::to_json(json & json, const DrvRef & ref ref); } -template struct adl_serializer>; -template struct adl_serializer>; +template struct adl_serializer>; +template struct adl_serializer>; } // namespace nlohmann diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 6238c8890499..d03b7f6fdb72 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -404,10 +404,9 @@ static RegisterStoreImplementation regDummyStore; namespace nlohmann { -using namespace nix; - -DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) +nix::DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); return DummyStore::PathInfoAndContents{ .info = valueAt(obj, "info"), @@ -415,7 +414,8 @@ DummyStore::PathInfoAndContents adl_serializer: }; } -void adl_serializer::to_json(json & json, const DummyStore::PathInfoAndContents & val) +void adl_serializer::to_json( + json & json, const nix::DummyStore::PathInfoAndContents & val) { json = { {"info", val.info}, @@ -423,8 +423,9 @@ void adl_serializer::to_json(json & json, const }; } -ref adl_serializer>::from_json(const json & json) +nix::ref adl_serializer>::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); auto cfg = make_ref(DummyStore::Config::Params{}); cfg->storeDir_.set(getString(valueAt(obj, "store"))); @@ -432,15 +433,16 @@ ref adl_serializer>::from_json(const j return cfg; } -void adl_serializer::to_json(json & json, const DummyStoreConfig & val) +void adl_serializer::to_json(json & json, const nix::DummyStoreConfig & val) { json = { {"store", val.storeDir}, }; } -ref adl_serializer>::from_json(const json & json) +nix::ref adl_serializer>::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); ref res = adl_serializer>::from_json(valueAt(obj, "config"))->openDummyStore(); for (auto & [k, v] : getObject(valueAt(obj, "contents"))) @@ -457,8 +459,9 @@ ref adl_serializer>::from_json(const json & json) return res; } -void adl_serializer::to_json(json & json, const DummyStore & val) +void adl_serializer::to_json(json & json, const nix::DummyStore & val) { + using namespace nix; json = { {"config", *val.config}, {"contents", diff --git a/src/libstore/include/nix/store/outputs-spec.hh b/src/libstore/include/nix/store/outputs-spec.hh index 5482c0e24bde..11613d7c84cf 100644 --- a/src/libstore/include/nix/store/outputs-spec.hh +++ b/src/libstore/include/nix/store/outputs-spec.hh @@ -140,5 +140,5 @@ struct ExtendedOutputsSpec } // namespace nix -JSON_IMPL(OutputsSpec) -JSON_IMPL(ExtendedOutputsSpec) +JSON_IMPL(nix::OutputsSpec) +JSON_IMPL(nix::ExtendedOutputsSpec) diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 5beeb0c070af..3689009fc1bc 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -135,4 +135,4 @@ NIX_DECLARE_CONFIG_SERIALISER(std::set) } // namespace nix -JSON_IMPL(StoreReference) +JSON_IMPL(nix::StoreReference) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 8db977b6fce0..1ae9dae3573b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -443,14 +443,13 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) namespace nlohmann { -using namespace nix; - -TrustedFlag adl_serializer::from_json(const json & json) +nix::TrustedFlag adl_serializer::from_json(const json & json) { + using namespace nix; return getBoolean(json) ? TrustedFlag::Trusted : TrustedFlag::NotTrusted; } -void adl_serializer::to_json(json & json, const TrustedFlag & trustedFlag) +void adl_serializer::to_json(json & json, const nix::TrustedFlag & trustedFlag) { json = static_cast(trustedFlag); } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 8a7ab0fae489..0c350138bb01 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -202,16 +202,14 @@ UnkeyedNarInfo UnkeyedNarInfo::fromJSON(const StoreDirConfig * store, const nloh namespace nlohmann { -using namespace nix; - -UnkeyedNarInfo adl_serializer::from_json(const json & json) +nix::UnkeyedNarInfo adl_serializer::from_json(const json & json) { - return UnkeyedNarInfo::fromJSON(nullptr, json); + return nix::UnkeyedNarInfo::fromJSON(nullptr, json); } -void adl_serializer::to_json(json & json, const UnkeyedNarInfo & c) +void adl_serializer::to_json(json & json, const nix::UnkeyedNarInfo & c) { - json = c.toJSON(nullptr, true, PathInfoJsonFormat::V2); + json = c.toJSON(nullptr, true, nix::PathInfoJsonFormat::V2); } } // namespace nlohmann diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 622df5fc3447..9f0f3c421a37 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -133,21 +133,21 @@ bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const namespace nlohmann { -using namespace nix; - #ifndef DOXYGEN_SKIP -OutputsSpec adl_serializer::from_json(const json & json) +nix::OutputsSpec adl_serializer::from_json(const json & json) { - auto names = json.get(); + using namespace nix; + auto names = json.get(); if (names == StringSet({"*"})) return OutputsSpec::All{}; else return OutputsSpec::Names{std::move(names)}; } -void adl_serializer::to_json(json & json, const OutputsSpec & t) +void adl_serializer::to_json(json & json, const nix::OutputsSpec & t) { + using namespace nix; std::visit( overloaded{ [&](const OutputsSpec::All &) { json = std::vector({"*"}); }, @@ -156,8 +156,9 @@ void adl_serializer::to_json(json & json, const OutputsSpec & t) t.raw); } -ExtendedOutputsSpec adl_serializer::from_json(const json & json) +nix::ExtendedOutputsSpec adl_serializer::from_json(const json & json) { + using namespace nix; if (json.is_null()) return ExtendedOutputsSpec::Default{}; else { @@ -165,8 +166,9 @@ ExtendedOutputsSpec adl_serializer::from_json(const json & } } -void adl_serializer::to_json(json & json, const ExtendedOutputsSpec & t) +void adl_serializer::to_json(json & json, const nix::ExtendedOutputsSpec & t) { + using namespace nix; std::visit( overloaded{ [&](const ExtendedOutputsSpec::Default &) { json = nullptr; }, diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 9408e9890900..a5c7314868d4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -306,30 +306,29 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig * store namespace nlohmann { -using namespace nix; - -PathInfoJsonFormat adl_serializer::from_json(const json & json) +nix::PathInfoJsonFormat adl_serializer::from_json(const json & json) { - return parsePathInfoJsonFormat(getUnsigned(json)); + return nix::parsePathInfoJsonFormat(nix::getUnsigned(json)); } -void adl_serializer::to_json(json & json, const PathInfoJsonFormat & format) +void adl_serializer::to_json(json & json, const nix::PathInfoJsonFormat & format) { json = static_cast(format); } -UnkeyedValidPathInfo adl_serializer::from_json(const json & json) +nix::UnkeyedValidPathInfo adl_serializer::from_json(const json & json) { - return UnkeyedValidPathInfo::fromJSON(nullptr, json); + return nix::UnkeyedValidPathInfo::fromJSON(nullptr, json); } -void adl_serializer::to_json(json & json, const UnkeyedValidPathInfo & c) +void adl_serializer::to_json(json & json, const nix::UnkeyedValidPathInfo & c) { - json = c.toJSON(nullptr, true, PathInfoJsonFormat::V3); + json = c.toJSON(nullptr, true, nix::PathInfoJsonFormat::V3); } -ValidPathInfo adl_serializer::from_json(const json & json0) +nix::ValidPathInfo adl_serializer::from_json(const json & json0) { + using namespace nix; auto json = getObject(json0); return ValidPathInfo{ @@ -338,8 +337,9 @@ ValidPathInfo adl_serializer::from_json(const json & json0) }; } -void adl_serializer::to_json(json & json, const ValidPathInfo & v) +void adl_serializer::to_json(json & json, const nix::ValidPathInfo & v) { + using namespace nix; adl_serializer::to_json(json, v); json["path"] = v.path; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 10520431105c..a3e1597b686c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -85,14 +85,12 @@ StorePath StorePath::random(std::string_view name) namespace nlohmann { -using namespace nix; - -StorePath adl_serializer::from_json(const json & json) +nix::StorePath adl_serializer::from_json(const json & json) { - return StorePath{getString(json)}; + return nix::StorePath{nix::getString(json)}; } -void adl_serializer::to_json(json & json, const StorePath & storePath) +void adl_serializer::to_json(json & json, const nix::StorePath & storePath) { json = storePath.to_string(); } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 443f97baa610..36de83a9bf82 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -97,10 +97,9 @@ MissingRealisation::MissingRealisation( namespace nlohmann { -using namespace nix; - -DrvOutput adl_serializer::from_json(const json & json) +nix::DrvOutput adl_serializer::from_json(const json & json) { + using namespace nix; auto obj = getObject(json); return { @@ -109,7 +108,7 @@ DrvOutput adl_serializer::from_json(const json & json) }; } -void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) +void adl_serializer::to_json(json & json, const nix::DrvOutput & drvOutput) { json = { {"drvPath", drvOutput.drvPath}, @@ -117,8 +116,9 @@ void adl_serializer::to_json(json & json, const DrvOutput & drvOutput }; } -UnkeyedRealisation adl_serializer::from_json(const json & json0) +nix::UnkeyedRealisation adl_serializer::from_json(const json & json0) { + using namespace nix; auto json = getObject(json0); return UnkeyedRealisation{ @@ -131,7 +131,7 @@ UnkeyedRealisation adl_serializer::from_json(const json & js }; } -void adl_serializer::to_json(json & json, const UnkeyedRealisation & r) +void adl_serializer::to_json(json & json, const nix::UnkeyedRealisation & r) { json = { {"outPath", r.outPath}, @@ -139,8 +139,9 @@ void adl_serializer::to_json(json & json, const UnkeyedReali }; } -Realisation adl_serializer::from_json(const json & json) +nix::Realisation adl_serializer::from_json(const json & json) { + using namespace nix; auto obj = getObject(json); return { @@ -149,11 +150,11 @@ Realisation adl_serializer::from_json(const json & json) }; } -void adl_serializer::to_json(json & json, const Realisation & r) +void adl_serializer::to_json(json & json, const nix::Realisation & r) { json = { {"key", r.id}, - {"value", static_cast(r)}, + {"value", static_cast(r)}, }; } diff --git a/src/libstore/s3-url.cc b/src/libstore/s3-url.cc index d6a18066f7aa..38d51ff5ac8b 100644 --- a/src/libstore/s3-url.cc +++ b/src/libstore/s3-url.cc @@ -11,8 +11,6 @@ #include #include -using namespace std::string_view_literals; - namespace nix { void InvalidS3AddressingStyle::anchor() {} @@ -43,6 +41,8 @@ std::string_view showS3AddressingStyle(S3AddressingStyle style) ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed) try { + using namespace std::string_view_literals; + if (parsed.scheme != "s3"sv) throw BadURL("URI scheme '%s' is not 's3'", parsed.scheme); diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 8f72a3d33081..029d8294cb19 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -189,14 +189,12 @@ std::pair splitUriAndParams(const std::stri namespace nlohmann { -using namespace nix; - -StoreReference adl_serializer::from_json(const json & json) +nix::StoreReference adl_serializer::from_json(const json & json) { - return StoreReference::parse(json.get()); + return nix::StoreReference::parse(json.get()); // FIXME: getString } -void adl_serializer::to_json(json & json, const StoreReference & ref) +void adl_serializer::to_json(json & json, const nix::StoreReference & ref) { json = ref.render(); } diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 64c541e3a9f9..be68f3044331 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -4,8 +4,6 @@ #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" -using namespace std::chrono_literals; - namespace nix { HookInstance::HookInstance(const Strings & _buildHook) @@ -72,6 +70,8 @@ HookInstance::HookInstance(const Strings & _buildHook) throw SysError("executing %s", PathFmt(buildHook)); }); + using namespace std::chrono_literals; + /* Give custom build hooks the chance to cleanup. */ pid.setKillSignal(SIGTERM); pid.setKillTimeout(500ms); diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index 2dc5da5f2c14..965097bf5295 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -7,10 +7,9 @@ namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; Hash prototype(HashAlgorithm::SHA1); return gen::apply( [](const std::vector & v) { diff --git a/src/libutil-test-support/include/nix/util/tests/hash.hh b/src/libutil-test-support/include/nix/util/tests/hash.hh index ee676f3eb210..d3efb5c3cd41 100644 --- a/src/libutil-test-support/include/nix/util/tests/hash.hh +++ b/src/libutil-test-support/include/nix/util/tests/hash.hh @@ -7,12 +7,10 @@ namespace rc { -using namespace nix; - template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libutil-tests/checked-arithmetic.cc b/src/libutil-tests/checked-arithmetic.cc index cc88d353932c..e0447b68e382 100644 --- a/src/libutil-tests/checked-arithmetic.cc +++ b/src/libutil-tests/checked-arithmetic.cc @@ -11,8 +11,6 @@ namespace rc { -using namespace nix; - template struct Arbitrary> { diff --git a/src/libutil-tests/closure.cc b/src/libutil-tests/closure.cc index 19df3e18f0c1..43b20823f252 100644 --- a/src/libutil-tests/closure.cc +++ b/src/libutil-tests/closure.cc @@ -3,9 +3,7 @@ namespace nix { -using namespace std; - -map> testGraph = { +std::map> testGraph = { {"A", {"B", "C", "G"}}, {"B", {"A"}}, // Loops back to A {"C", {"F"}}, // Indirect reference @@ -17,9 +15,9 @@ map> testGraph = { TEST(closure, correctClosure) { - set aClosure; - set expectedClosure = {"A", "B", "C", "F", "G"}; - computeClosure( + std::set aClosure; + std::set expectedClosure = {"A", "B", "C", "F", "G"}; + computeClosure( {"A"}, aClosure, [&](const std::string & currentNode) -> asio::awaitable> { co_return testGraph[currentNode]; }); @@ -32,10 +30,10 @@ TEST(closure, properlyHandlesDirectExceptions) struct TestExn {}; - set aClosure; + std::set aClosure; std::size_t callCount = 0; EXPECT_THROW( - computeClosure( + computeClosure( {"A", "B"}, aClosure, [&](const std::string &) -> asio::awaitable> { diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 417622a47cf2..c89cd221d199 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -5,8 +5,6 @@ #include #include -using namespace std::string_view_literals; - #ifdef _WIN32 # define FS_SEP L"\\" # define FS_ROOT_NO_TRAILING_SLASH L"C:" // Need a mounted one, C drive is likely @@ -111,6 +109,8 @@ TEST(canonPath, removesDots2) TEST(canonPath, requiresAbsolutePath) { + using namespace std::string_view_literals; + ASSERT_ANY_THROW(canonPath("."sv)); ASSERT_ANY_THROW(canonPath(".."sv)); ASSERT_ANY_THROW(canonPath("../"sv)); diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc index e80bfaeb9adb..bcdc70522b69 100644 --- a/src/libutil-tests/memory-source-accessor.cc +++ b/src/libutil-tests/memory-source-accessor.cc @@ -11,7 +11,6 @@ namespace nix { namespace memory_source_accessor { -using namespace std::literals; using File = MemorySourceAccessor::File; ref exampleSimple() @@ -26,6 +25,7 @@ ref exampleSimple() ref exampleComplex() { + using namespace std::literals; auto files = make_ref(); files->root = File::Directory{ .entries{ diff --git a/src/libutil/base-n.cc b/src/libutil/base-n.cc index 4c9726ad2e5e..d5553a87e262 100644 --- a/src/libutil/base-n.cc +++ b/src/libutil/base-n.cc @@ -4,8 +4,6 @@ #include "nix/util/util.hh" #include "nix/util/base-n.hh" -using namespace std::literals; - namespace nix { constexpr static const std::array base16Chars = "0123456789abcdef"_arrayNoNull; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 96c6dd28791d..5cc1150aed7a 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -12,9 +12,6 @@ namespace nix::git { -using namespace nix; -using namespace std::string_literals; - std::optional decodeMode(RawMode m) { switch (m) { @@ -230,6 +227,7 @@ void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlg void dumpBlobPrefix(uint64_t size, Sink & sink, const ExperimentalFeatureSettings & xpSettings) { + using namespace std::string_literals; xpSettings.require(Xp::GitHashing); auto s = fmt("blob %d\0"s, std::to_string(size)); sink(s); @@ -237,6 +235,7 @@ void dumpBlobPrefix(uint64_t size, Sink & sink, const ExperimentalFeatureSetting void dumpTree(const Tree & entries, Sink & sink, const ExperimentalFeatureSettings & xpSettings) { + using namespace std::string_literals; xpSettings.require(Xp::GitHashing); std::string v1; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index ee5f28ffea5c..a50b1bd3d9e2 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -519,17 +519,15 @@ std::string_view printHashAlgo(HashAlgorithm ha) namespace nlohmann { -using namespace nix; - -Hash adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) +nix::Hash adl_serializer::from_json(const json & json, const nix::ExperimentalFeatureSettings & xpSettings) { - auto & s = getString(json); - return Hash::parseSRI(s, xpSettings); + auto & s = nix::getString(json); + return nix::Hash::parseSRI(s, xpSettings); } -void adl_serializer::to_json(json & json, const Hash & hash) +void adl_serializer::to_json(json & json, const nix::Hash & hash) { - json = hash.to_string(HashFormat::SRI, true); + json = hash.to_string(nix::HashFormat::SRI, true); } } // namespace nlohmann diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index 2dab25ea9afb..0ea54c802b16 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -265,4 +265,4 @@ inline std::size_t hash_value(const Hash & hash) } // namespace nix -JSON_IMPL_WITH_XP_FEATURES(Hash) +JSON_IMPL_WITH_XP_FEATURES(nix::Hash) diff --git a/src/libutil/include/nix/util/json-impls.hh b/src/libutil/include/nix/util/json-impls.hh index 26a94472f25e..5a1c1354309a 100644 --- a/src/libutil/include/nix/util/json-impls.hh +++ b/src/libutil/include/nix/util/json-impls.hh @@ -27,19 +27,18 @@ #define JSON_IMPL(TYPE) \ namespace nlohmann { \ - using namespace nix; \ template<> \ JSON_IMPL_INNER(TYPE); \ } -#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \ - namespace nlohmann { \ - using namespace nix; \ - template<> \ - struct adl_serializer \ - { \ - static TYPE \ - from_json(const json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); \ - static void to_json(json & json, const TYPE & t); \ - }; \ +#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \ + namespace nlohmann { \ + template<> \ + struct adl_serializer \ + { \ + static TYPE from_json( \ + const json & json, \ + const nix::ExperimentalFeatureSettings & xpSettings = nix::experimentalFeatureSettings); \ + static void to_json(json & json, const TYPE & t); \ + }; \ } diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index 6e1792abc3a3..80be8a335f0e 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -194,29 +194,27 @@ struct json_avoids_null : std::true_type namespace nlohmann { -using namespace nix; - -#define ARG fso::Regular +#define ARG nix::fso::Regular template JSON_IMPL_INNER(ARG); #undef ARG -#define ARG fso::DirectoryT +#define ARG nix::fso::DirectoryT template JSON_IMPL_INNER(ARG); #undef ARG template<> -JSON_IMPL_INNER(fso::Symlink); +JSON_IMPL_INNER(nix::fso::Symlink); template<> -JSON_IMPL_INNER(fso::Opaque); +JSON_IMPL_INNER(nix::fso::Opaque); -#define ARG fso::VariantT +#define ARG nix::fso::VariantT template JSON_IMPL_INNER(ARG); #undef ARG } // namespace nlohmann -JSON_IMPL(MemorySourceAccessor) +JSON_IMPL(nix::MemorySourceAccessor) diff --git a/src/libutil/memory-source-accessor/json.cc b/src/libutil/memory-source-accessor/json.cc index ff3808d3c363..e9908035bdb8 100644 --- a/src/libutil/memory-source-accessor/json.cc +++ b/src/libutil/memory-source-accessor/json.cc @@ -6,22 +6,22 @@ namespace nlohmann { -using namespace nix; - // fso::Regular template<> -MemorySourceAccessor::File::Regular adl_serializer::from_json(const json & json) +nix::MemorySourceAccessor::File::Regular +adl_serializer::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); - return MemorySourceAccessor::File::Regular{ + return { .executable = getBoolean(valueAt(obj, "executable")), .contents = getString(valueAt(obj, "contents")), }; } template<> -void adl_serializer::to_json( - json & json, const MemorySourceAccessor::File::Regular & r) +void adl_serializer::to_json( + json & json, const nix::MemorySourceAccessor::File::Regular & r) { json = { {"executable", r.executable}, @@ -30,8 +30,9 @@ void adl_serializer::to_json( } template<> -NarListing::Regular adl_serializer::from_json(const json & json) +nix::NarListing::Regular adl_serializer::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); auto * execPtr = optionalValueAt(obj, "executable"); auto * sizePtr = optionalValueAt(obj, "size"); @@ -47,7 +48,7 @@ NarListing::Regular adl_serializer::from_json(const json & } template<> -void adl_serializer::to_json(json & j, const NarListing::Regular & r) +void adl_serializer::to_json(json & j, const nix::NarListing::Regular & r) { if (r.contents.fileSize) j["size"] = *r.contents.fileSize; @@ -57,30 +58,32 @@ void adl_serializer::to_json(json & j, const NarListing::Re } template -void adl_serializer>::to_json(json & j, const fso::DirectoryT & d) +void adl_serializer>::to_json(json & j, const nix::fso::DirectoryT & d) { j["entries"] = d.entries; } template -fso::DirectoryT adl_serializer>::from_json(const json & json) +nix::fso::DirectoryT adl_serializer>::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); - return fso::DirectoryT{ + return { .entries = valueAt(obj, "entries"), }; } // fso::Symlink -fso::Symlink adl_serializer::from_json(const json & json) +nix::fso::Symlink adl_serializer::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); - return fso::Symlink{ + return { .target = getString(valueAt(obj, "target")), }; } -void adl_serializer::to_json(json & json, const fso::Symlink & s) +void adl_serializer::to_json(json & json, const nix::fso::Symlink & s) { json = { {"target", s.target}, @@ -88,21 +91,22 @@ void adl_serializer::to_json(json & json, const fso::Symlink & s) } // fso::Opaque -fso::Opaque adl_serializer::from_json(const json &) +nix::fso::Opaque adl_serializer::from_json(const json &) { - return fso::Opaque{}; + return {}; } -void adl_serializer::to_json(json & j, const fso::Opaque &) +void adl_serializer::to_json(json & j, const nix::fso::Opaque &) { j = nlohmann::json::object(); } // fso::VariantT - generic implementation template -void adl_serializer>::to_json( - json & j, const fso::VariantT & val) +void adl_serializer>::to_json( + json & j, const nix::fso::VariantT & val) { + using namespace nix; using Variant = fso::VariantT; j = nlohmann::json::object(); std::visit( @@ -124,9 +128,10 @@ void adl_serializer>::to_json( } template -fso::VariantT -adl_serializer>::from_json(const json & json) +nix::fso::VariantT +adl_serializer>::from_json(const json & json) { + using namespace nix; using Variant = fso::VariantT; auto & obj = getObject(json); auto type = getString(valueAt(obj, "type")); @@ -141,19 +146,19 @@ adl_serializer>::from_json(const json & js } // Explicit instantiations for VariantT types we use -template struct adl_serializer; -template struct adl_serializer; -template struct adl_serializer; +template struct adl_serializer; +template struct adl_serializer; +template struct adl_serializer; // MemorySourceAccessor -MemorySourceAccessor adl_serializer::from_json(const json & json) +nix::MemorySourceAccessor adl_serializer::from_json(const json & json) { - MemorySourceAccessor res; + nix::MemorySourceAccessor res; res.root = json; return res; } -void adl_serializer::to_json(json & json, const MemorySourceAccessor & val) +void adl_serializer::to_json(json & json, const nix::MemorySourceAccessor & val) { json = val.root; } diff --git a/src/libutil/signature/local-keys.cc b/src/libutil/signature/local-keys.cc index 8afa3eb728f2..74b183b7e52a 100644 --- a/src/libutil/signature/local-keys.cc +++ b/src/libutil/signature/local-keys.cc @@ -176,16 +176,17 @@ bool verifyDetached(std::string_view data, const Signature & sig, const PublicKe } // namespace nix namespace nlohmann { -void adl_serializer::to_json(json & j, const Signature & s) +void adl_serializer::to_json(json & j, const nix::Signature & s) { j = { {"keyName", s.keyName}, - {"sig", base64::encode(std::as_bytes(std::span{s.sig}))}, + {"sig", nix::base64::encode(std::as_bytes(std::span{s.sig}))}, }; } -Signature adl_serializer::from_json(const json & j) +nix::Signature adl_serializer::from_json(const json & j) { + using namespace nix; if (j.is_string()) return Signature::parse(getString(j)); auto obj = getObject(j); diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 32d59535883a..0b1ab51dd82c 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -14,7 +14,6 @@ #include #include #include -using namespace std::chrono_literals; #include #include @@ -80,6 +79,8 @@ int Pid::kill(bool allowInterrupts) std::atomic killed = false; + using namespace std::chrono_literals; + if (killTimeout > 0ms && killSignal != SIGKILL) killThread = std::thread([&]() { auto elapsed = 0ms; diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index 785231a0f6df..c485a084746c 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -31,8 +31,6 @@ #include "nix/util/fun.hh" #include "man-pages.hh" -using namespace std::string_literals; - extern char ** environ __attribute__((weak)); namespace nix { @@ -636,6 +634,9 @@ static void main_nix_build(int argc, char ** argv) auto rcfile = (tmpDir.path() / "rc").string(); auto tz = getEnv("TZ"); auto tzExport = tz ? "export TZ=" + escapeShellArgAlways(*tz) + "; " : ""; + + using namespace std::string_literals; + std::string rc = fmt( (R"(_nix_shell_clean_tmpdir() { command rm -rf %1%; };)"s "trap _nix_shell_clean_tmpdir EXIT; " diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 2bb56eb51bca..77c251944f38 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -40,7 +40,7 @@ namespace nix_store { -using namespace nix; +using namespace nix; // NOLINT(nix-using-namespace) typedef void (*Operation)(Strings opFlags, Strings opArgs); From eb74c1691a7106f33debf4f3606ced4fd040cb64 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Jul 2026 00:41:11 -0400 Subject: [PATCH 343/364] Check `experimentalFeature()` of nested subcommands Previously only the top-level subcommand's `experimentalFeature()` was checked, so an override on a nested command (e.g. `nix store roots-daemon`'s `Xp::LocalOverlayStore`) was only reflected in the generated documentation, never actually enforced at runtime. Walk down the whole command chain, requiring each level's feature, the same way the `--help` code walks it just above. Assisted-by: Claude:fable-5 --- src/nix/main.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 963bc07ce274..a9300ddcb811 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -544,7 +544,13 @@ void mainWrapped(int argc, char ** argv) if (!args.command) throw UsageError("no subcommand specified"); - experimentalFeatureSettings.require(args.command->second->experimentalFeature()); + { + MultiCommand * command = &args; + while (command && command->command) { + experimentalFeatureSettings.require(command->command->second->experimentalFeature()); + command = dynamic_cast(&*command->command->second); + } + } if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); From 440dcf4858188f5c94898fede30a337d62b83d54 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Jul 2026 00:42:59 -0400 Subject: [PATCH 344/364] Rename `CmdRealisation*` structs to `CmdBuildTrace*` The command itself was already renamed from `nix realisation` to `nix store build-trace`; catch the C++ identifiers up with that. No functional change. Assisted-by: Claude:fabel-5 --- src/nix/build-trace.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/build-trace.cc b/src/nix/build-trace.cc index db5942226d49..d33ead492441 100644 --- a/src/nix/build-trace.cc +++ b/src/nix/build-trace.cc @@ -5,9 +5,9 @@ namespace nix { -struct CmdRealisation : NixMultiCommand +struct CmdBuildTrace : NixMultiCommand { - CmdRealisation() + CmdBuildTrace() : NixMultiCommand("build-trace", RegisterCommand::getCommandsFor({"store", "build-trace"})) { } @@ -23,9 +23,9 @@ struct CmdRealisation : NixMultiCommand } }; -static auto rCmdRealisation = registerCommand2({"store", "build-trace"}); +static auto rCmdBuildTrace = registerCommand2({"store", "build-trace"}); -struct CmdRealisationInfo : BuiltPathsCommand, MixJSON +struct CmdBuildTraceInfo : BuiltPathsCommand, MixJSON { std::string description() override { @@ -77,6 +77,6 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON } }; -static auto rCmdBuildTraceInfo = registerCommand2({"store", "build-trace", "info"}); +static auto rCmdBuildTraceInfo = registerCommand2({"store", "build-trace", "info"}); } // namespace nix From 04114b308fdbae0bb3f4fa95c4a1daeab1906718 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Dec 2025 19:33:04 -0500 Subject: [PATCH 345/364] Add `nix store build-trace delete` command Can be used to delete build traces/realisations. Also include a basic test to show functionality --- src/libstore/include/nix/store/gc-store.hh | 7 +++ src/libstore/include/nix/store/local-store.hh | 2 + .../include/nix/store/remote-store.hh | 6 +++ src/libstore/local-store.cc | 24 +++++++++ src/libstore/restricted-store.cc | 2 + src/nix/build-trace.cc | 51 ++++++++++++++++++- src/nix/build-trace/delete.md | 14 +++++ tests/functional/ca/build-trace-delete.sh | 35 +++++++++++++ tests/functional/ca/meson.build | 1 + tests/functional/ca/nondeterministic-ns.nix | 44 ++++++++++++++++ 10 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 src/nix/build-trace/delete.md create mode 100644 tests/functional/ca/build-trace-delete.sh create mode 100644 tests/functional/ca/nondeterministic-ns.nix diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 0a0335016b74..d7637374157f 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -135,6 +135,13 @@ public: * Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /** + * Delete build trace entries (realisations) from the store's database. + * + * The entries are specified by their key (the build trace is a map). + */ + virtual void deleteBuildTraces(const std::set & keys) = 0; }; } // namespace nix diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 0851795ff3d2..52a3ad5a9264 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -347,6 +347,8 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + void deleteBuildTraces(const std::set & keys) override; + /** * Called by `collectGarbage` to trace in reverse. * diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 144b4b8e4355..aa396d3ff7fd 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -132,6 +132,12 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + void deleteBuildTraces(const std::set & keys) override + { + // TODO support this in the protocol someday + unsupported("deleteBuildTraces"); + }; + void optimiseStore() override; bool verifyStore(bool checkContents, RepairFlag repair) override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 909f22369dc0..afa1b2e026c5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -31,6 +31,7 @@ #include #include #include +#include #ifndef _WIN32 # include @@ -112,6 +113,7 @@ struct LocalStore::State::Stmts SQLiteStmt AddDerivationOutput; SQLiteStmt RegisterRealisedOutput; SQLiteStmt UpdateRealisedOutput; + SQLiteStmt DeleteRealisedOutputByName; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; SQLiteStmt QueryRealisedOutput; @@ -389,6 +391,15 @@ LocalStore::LocalStore(ref config) outputName = ? ; )"); + state->stmts->DeleteRealisedOutputByName.create( + state->db, + R"( + delete from BuildTraceV3 + where + drvPath = ? and + outputName = ? + ; + )"); state->stmts->QueryRealisedOutput.create( state->db, R"( @@ -696,6 +707,19 @@ void LocalStore::registerDrvOutput(const Realisation & info) }); } +void LocalStore::deleteBuildTraces(const std::set & keys) +{ + experimentalFeatureSettings.require(Xp::CaDerivations); + retrySQLite([&]() { + auto state(_state->lock()); + SQLiteTxn txn(state->db); + for (const auto & key : keys) { + state->stmts->DeleteRealisedOutputByName.use().apply(key.drvPath.to_string()).apply(key.outputName).exec(); + } + txn.commit(); + }); +} + void LocalStore::cacheDrvOutputMapping( State & state, const uint64_t deriver, const std::string & outputName, const StorePath & output) { diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 1cc0f57af1f1..38ab658728bc 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -140,6 +140,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void collectGarbage(const GCOptions & options, GCResults & results) override {} + void deleteBuildTraces(const std::set & keys) override {} + void addSignatures(const StorePath & storePath, const std::set & sigs) override { unsupported("addSignatures"); diff --git a/src/nix/build-trace.cc b/src/nix/build-trace.cc index d33ead492441..00f22b78ee41 100644 --- a/src/nix/build-trace.cc +++ b/src/nix/build-trace.cc @@ -1,7 +1,10 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" +#include "nix/store/gc-store.hh" +#include "nix/store/store-cast.hh" #include +#include namespace nix { @@ -21,6 +24,11 @@ struct CmdBuildTrace : NixMultiCommand { return catUtility; } + + std::optional experimentalFeature() override + { + return Xp::CaDerivations; + } }; static auto rCmdBuildTrace = registerCommand2({"store", "build-trace"}); @@ -46,7 +54,6 @@ struct CmdBuildTraceInfo : BuiltPathsCommand, MixJSON void run(ref store, BuiltPaths && paths, BuiltPaths && rootPaths) override { - experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; for (auto & builtPath : paths) { @@ -79,4 +86,46 @@ struct CmdBuildTraceInfo : BuiltPathsCommand, MixJSON static auto rCmdBuildTraceInfo = registerCommand2({"store", "build-trace", "info"}); +struct CmdBuildTraceDelete : virtual StoreCommand +{ + std::vector ids; + + CmdBuildTraceDelete() + { + expectArgs({ + .label = "id", + .handler = {&ids}, + }); + } + + std::string description() override + { + return "delete build traces from the store"; + } + + std::string doc() override + { + return +#include "build-trace/delete.md" + ; + } + + Category category() override + { + return catSecondary; + } + + void run(ref store) override + { + auto & gcStore = require(*store); + + auto keys = ids | std::views::transform([&](std::string_view s) { return DrvOutput::parse(*store, s); }) + | std::ranges::to(); + + gcStore.deleteBuildTraces(keys); + } +}; + +static auto rCmdBuildTraceDelete = registerCommand2({"store", "build-trace", "delete"}); + } // namespace nix diff --git a/src/nix/build-trace/delete.md b/src/nix/build-trace/delete.md new file mode 100644 index 000000000000..6200a06daa80 --- /dev/null +++ b/src/nix/build-trace/delete.md @@ -0,0 +1,14 @@ +R"MdBoundary( +# Description + +Delete build traces from the store. + +# Examples + +Delete a build trace by its ID: + +```console +$ nix store build-trace delete /nix/store/wfyy9qiwph8zxk68g4p71nrxi0k59x5y-python3-minimal-3.13.12.drv^debug +``` + +)MdBoundary" diff --git a/tests/functional/ca/build-trace-delete.sh b/tests/functional/ca/build-trace-delete.sh new file mode 100644 index 000000000000..54caaca01a03 --- /dev/null +++ b/tests/functional/ca/build-trace-delete.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +source common.sh + +set -x + +needLocalStore "command 'nix store build-trace delete' can’t be used with the daemon" + +clearStore + +singleOutput=$(nix-instantiate ./nondeterministic-ns.nix -A singleOut) +multiOutput=$(nix-instantiate ./nondeterministic-ns.nix -A multiOut) + +# First build +singleOutPath=$(nix-build ./nondeterministic-ns.nix -A singleOut --no-out-link) +nix-store --delete "$singleOutPath" +# We should still have the build trace/realisation in the database, so second build will fail +expect 1 nix-build ./nondeterministic-ns.nix -A singleOut --no-out-link +# Deleting the build trace/realisation should fix it though +nix store build-trace delete "$singleOutput"^out +nix-build ./nondeterministic-ns.nix -A singleOut --no-out-link + +# Multi-output first +nix-build ./nondeterministic-ns.nix -A multiOut --no-out-link +multiOutPath=$(nix store build-trace info "$multiOutput"^out --json | jq -r '.[] | .opaquePath | select(.)') +multiLibPath=$(nix store build-trace info "$multiOutput"^lib --json | jq -r '.[] | .opaquePath | select(.)') + +# We should be able to delete multiple build traces/realisations at once +nix-store --delete "$multiOutPath" "$multiLibPath" +nix store build-trace delete "$multiOutput"^out "$multiOutput"^lib + +# out and lib should be deleted, but dev should not +expect 1 nix store build-trace info "$multiOutput"^out +expect 1 nix store build-trace info "$multiOutput"^lib +nix store build-trace info "$multiOutput"^dev diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build index c60db18534a1..2625388e4eb3 100644 --- a/tests/functional/ca/meson.build +++ b/tests/functional/ca/meson.build @@ -4,6 +4,7 @@ suites += { 'tests' : [ 'build-cache.sh', 'build-delete.sh', + 'build-trace-delete.sh', 'build-with-garbage-path.sh', 'build.sh', 'concurrent-builds.sh', diff --git a/tests/functional/ca/nondeterministic-ns.nix b/tests/functional/ca/nondeterministic-ns.nix new file mode 100644 index 000000000000..899db8cb5ca0 --- /dev/null +++ b/tests/functional/ca/nondeterministic-ns.nix @@ -0,0 +1,44 @@ +with import ./config.nix; + +let + mkCADerivation = + args: + mkDerivation ( + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + // args + ); +in + +{ + singleOut = mkCADerivation { + name = "time-single-out"; + buildCommand = '' + mkdir $out + date +%s.%N > $out/current-time + ''; + }; + + multiOut = mkCADerivation { + name = "time-multi-out"; + outputs = [ + "out" + "lib" + "dev" + ]; + buildCommand = '' + mkdir $out + date +%s.%N > $out/current-time + echo out > $out/foo + mkdir $lib + date +%s.%N > $lib/current-time + echo lib > $lib/foo + mkdir $dev + date +%s.%N > $lib/current-time + echo dev > $dev/foo + ''; + }; +} From c93fac6b7496cb9b0dce69af6a29c41c9e507b20 Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Wed, 13 May 2026 13:25:54 -0400 Subject: [PATCH 346/364] Add AddToStoreScanning operation --- src/libstore/content-address.cc | 15 ++++++ src/libstore/daemon.cc | 37 +++++++++++++- .../include/nix/store/content-address.hh | 9 ++++ src/libstore/include/nix/store/local-store.hh | 13 +++++ src/libstore/include/nix/store/meson.build | 1 + .../include/nix/store/remote-store.hh | 10 +++- .../include/nix/store/submit-store.hh | 31 ++++++++++++ .../include/nix/store/worker-protocol.hh | 6 +++ src/libstore/local-store.cc | 37 ++++++++++++-- src/libstore/meson.build | 1 + src/libstore/remote-store.cc | 50 ++++++++++++------- src/libstore/restricted-store.cc | 24 ++++++++- src/libstore/submit-store.cc | 7 +++ 13 files changed, 216 insertions(+), 25 deletions(-) create mode 100644 src/libstore/include/nix/store/submit-store.hh create mode 100644 src/libstore/submit-store.cc diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 2c40e6f6dd31..fd8807d6d035 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,5 +1,6 @@ #include "nix/util/args.hh" #include "nix/store/content-address.hh" +#include "nix/util/file-content-address.hh" #include "nix/util/split.hh" #include "nix/util/json-utils.hh" @@ -133,6 +134,20 @@ FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const } } +FileSerialisationMethod ContentAddressMethod::getFileSerialisationMethod() const +{ + switch (raw) { + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::Text: + return FileSerialisationMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FileSerialisationMethod::NixArchive; + default: + assert(false); + } +} + std::string ContentAddress::render() const { return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 563a5bafd030..bac95818ff46 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,4 +1,5 @@ #include "nix/store/daemon.hh" +#include "nix/util/file-content-address.hh" #include "nix/util/signals.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" @@ -11,6 +12,7 @@ #include "nix/store/indirect-root-store.hh" #include "nix/store/remote-store.hh" #include "nix/store/path-with-outputs.hh" +#include "nix/store/submit-store.hh" #include "nix/util/finally.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" @@ -1014,6 +1016,37 @@ static void performOp( break; } + case WorkerProto::Op::AddToStoreScanning: { + auto name = readString(conn.from); + auto camStr = readString(conn.from); + + experimentalFeatureSettings.require(Xp::DynamicDerivations); + + if (!conn.protoVersion.features.contains(WorkerProto::featureAddToStoreScanning)) + throw Error("Adding to store with scanning was requested, but not supported in negotiated protocol"); + + if (!recursive) + throw Error("AddToStoreScanning only valid inside a `recursive-nix` derivation builder"); + + auto & submitStore = require(*store); + + logger->startWork(); + auto pathInfo = [&]() { + // NB: FramedSource must be out of scope before logger->stopWork(); + // FIXME: this means that if there is an error + // half-way through, the client will keep sending + // data, since we haven't sent it the error yet. + auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); + FramedSource source(conn.from); + FileSerialisationMethod dumpMethod = contentAddressMethod.getFileSerialisationMethod(); + return submitStore.addToStoreScanning(source, name, dumpMethod, contentAddressMethod, hashAlgo); + }(); + logger->stopWork(); + + WorkerProto::Serialise::write(*store, wconn, *pathInfo); + break; + } + default: throw Error("invalid operation %1%", op); } @@ -1039,8 +1072,10 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted /* Exchange the greeting. */ auto localVersion = WorkerProto::latest; - if (recursive) + if (recursive) { localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); + localVersion.features.insert(std::string{WorkerProto::featureAddToStoreScanning}); + } WorkerProto::BasicServerConnection conn; conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, localVersion); diff --git a/src/libstore/include/nix/store/content-address.hh b/src/libstore/include/nix/store/content-address.hh index 41ccc69aeb3f..ce700bd1ac0d 100644 --- a/src/libstore/include/nix/store/content-address.hh +++ b/src/libstore/include/nix/store/content-address.hh @@ -131,6 +131,15 @@ struct ContentAddressMethod * for hashing file systeme objects. */ FileIngestionMethod getFileIngestionMethod() const; + + /** + * The FileSerialisationMethod that is recommended for this content addressing method. + * In some circumstances, other methods are also valid. + * Note that `Git` is mapped to `NixArchive`, even though it could represent a single file. + * There is no support for flat serialisation of merkle objects; and even if there were, + * it would be unable to represent executable files. + */ + FileSerialisationMethod getFileSerialisationMethod() const; }; /* diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 52a3ad5a9264..cbd4cc0c2dfb 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -304,6 +304,19 @@ public: const StorePathSet & references, RepairFlag repair) override; + // Designed to be used from RestrictedStore, + // allows filtering the references while scanning. + // Not an entirely separate function in order to reduce duplication + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair, + bool filterReferences); + void addTempRoot(const StorePath & path) override; private: diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 3ccf0adc2fa1..5a3d21e106ea 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -93,6 +93,7 @@ headers = [ config_pub_h ] + files( 'store-open.hh', 'store-reference.hh', 'store-registration.hh', + 'submit-store.hh', 'uds-remote-store.hh', 'worker-protocol-connection.hh', 'worker-protocol-impl.hh', diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index aa396d3ff7fd..14e8af3b00dd 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -6,6 +6,7 @@ #include #include "nix/store/store-api.hh" +#include "nix/store/submit-store.hh" #include "nix/util/sync.hh" #include "nix/util/file-descriptor.hh" #include "nix/store/gc-store.hh" @@ -46,7 +47,7 @@ public: * \todo RemoteStore is a misnomer - should be something like * DaemonStore. */ -struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore +struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore, public virtual SubmitStore { private: void anchor() override; @@ -113,6 +114,13 @@ public: void registerDrvOutput(const Realisation & info) override; + ref addToStoreScanning( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo) override; + void queryRealisationUncached( const DrvOutput &, Callback> callback) noexcept override; diff --git a/src/libstore/include/nix/store/submit-store.hh b/src/libstore/include/nix/store/submit-store.hh new file mode 100644 index 000000000000..48bea62faa8c --- /dev/null +++ b/src/libstore/include/nix/store/submit-store.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "nix/store/store-api.hh" + +namespace nix { + +struct SubmitStore : public virtual Store +{ +private: + void anchor() override; + +public: + inline static std::string operationName = "Submit outputs for a currently running derivation"; + + /** + * Add to store, scanning references. + * Only within a recursive-nix derivation, as there would otherwise be no known + * set of valid store paths + */ + virtual ref addToStoreScanning( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256) = 0; + + static SubmitStore & require(Store & store); +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index d09d30421dcf..a7dd7da65118 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -135,6 +135,11 @@ struct WorkerProto */ static constexpr std::string_view featureDisableSetOptions = "disable-set-options"; + /** + * Feature for enabling the `AddToStoreScanning` operation + */ + static constexpr std::string_view featureAddToStoreScanning = "add-to-store-scanning"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -256,6 +261,7 @@ enum struct WorkerProto::Op : uint64_t { AddBuildLog = 45, BuildPathsWithResults = 46, AddPermRoot = 47, + AddToStoreScanning = 1001, }; struct WorkerProto::ClientHandshakeInfo diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index afa1b2e026c5..f66099005697 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,5 +1,6 @@ #include "nix/store/local-store.hh" #include "nix/store/globals.hh" +#include "nix/store/path-references.hh" #include "nix/util/git.hh" #include "nix/util/archive.hh" #include "nix/store/pathlocks.hh" @@ -1180,10 +1181,30 @@ StorePath LocalStore::addToStoreFromDump( HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) +{ + return addToStoreFromDump(source0, name, dumpMethod, hashMethod, hashAlgo, references, repair, false); +} + +StorePath LocalStore::addToStoreFromDump( + Source & source0, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & originalReferences, + RepairFlag repair, + bool filterReferences) { /* For computing the store path. */ - auto hashSink = std::make_unique(hashAlgo); - TeeSource source{source0, *hashSink}; + auto hashSink = std::make_shared(hashAlgo); + std::shared_ptr sink = hashSink; + std::optional refSink = std::nullopt; + if (filterReferences) { + // Only scan if we really need to, since it's slower. + refSink = PathRefScanSink::fromPaths(originalReferences); + sink = std::make_shared(*hashSink, *refSink); + } + TeeSource source{source0, *sink}; const LocalSettings & localSettings = config->getLocalSettings(); /* Read the source path into memory, but only if it's up to @@ -1236,8 +1257,9 @@ StorePath LocalStore::addToStoreFromDump( bool methodsMatch = static_cast(dumpMethod) == hashMethod.getFileIngestionMethod(); /* If the methods don't match, our streaming hash of the dump is the - wrong sort, and we need to rehash. */ - bool inMemoryAndDontNeedRestore = inMemory && methodsMatch; + wrong sort, and we need to rehash. + References are also in store path, if scanning we will need to move */ + bool inMemoryAndDontNeedRestore = inMemory && methodsMatch && !filterReferences; if (!inMemoryAndDontNeedRestore) { /* Drain what we pulled so far, and then keep on pulling */ @@ -1256,6 +1278,13 @@ StorePath LocalStore::addToStoreFromDump( auto [dumpHash, size] = hashSink->finish(); + StorePathSet references; + if (refSink.has_value()) { + references = refSink->getResultPaths(); + } else { + references = originalReferences; + } + auto desc = ContentAddressWithReferences::fromParts( hashMethod, methodsMatch ? dumpHash diff --git a/src/libstore/meson.build b/src/libstore/meson.build index c266f05d4dd1..e2f3f4e92711 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -356,6 +356,7 @@ sources = files( 'store-dir-config.cc', 'store-reference.cc', 'store-registration.cc', + 'submit-store.cc', 'uds-remote-store.cc', 'worker-protocol-connection.cc', 'worker-protocol.cc', diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f0f672038bb9..bd8aa9163ea2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "nix/store/path.hh" #include "nix/store/store-api.hh" +#include "nix/util/file-content-address.hh" #include "nix/util/serialise.hh" #include "nix/util/util.hh" #include "nix/store/path-with-outputs.hh" @@ -89,10 +90,12 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - // The DisableSetOptions feature isn't in the `latest` constant because it is shared with the daemon, - // which only adds the feature under certain conditions. Adding is easier than removing. + // The DisableSetOptions and `AddToStoreScanning` features aren't in the `latest` constant because it is + // shared with the daemon, which only adds the feature under certain conditions. + // Adding is easier than removing. auto localVersion = WorkerProto::latest; localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); + localVersion.features.insert(std::string{WorkerProto::featureAddToStoreScanning}); conn.protoVersion = WorkerProto::BasicClientConnection::handshake(conn.to, tee, localVersion); if (conn.protoVersion.number < WorkerProto::minimum.number) @@ -422,22 +425,7 @@ StorePath RemoteStore::addToStoreFromDump( const StorePathSet & references, RepairFlag repair) { - FileSerialisationMethod fsm; - switch (hashMethod.getFileIngestionMethod()) { - case FileIngestionMethod::Flat: - fsm = FileSerialisationMethod::Flat; - break; - case FileIngestionMethod::NixArchive: - fsm = FileSerialisationMethod::NixArchive; - break; - case FileIngestionMethod::Git: - // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::NixArchive; - break; - default: - assert(false); - } - if (fsm != dumpMethod) + if (hashMethod.getFileSerialisationMethod() != dumpMethod) unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination"); auto storePath = addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path; invalidatePathInfoCacheFor(storePath); @@ -517,6 +505,32 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn.processStderr(); } +ref RemoteStore::addToStoreScanning( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo) +{ + if (hashMethod.getFileSerialisationMethod() != dumpMethod) + unsupported("RemoteStore::addToStoreScanning doesn't support this `dumpMethod` `hashMethod` combination"); + + auto conn(getConnection()); + if (!conn->protoVersion.features.contains(WorkerProto::featureAddToStoreScanning)) + throw Error("the daemon does not support AddToStoreScanning, perhaps this is not in a recursive-nix builder?"); + + conn->to << WorkerProto::Op::AddToStoreScanning << name << hashMethod.renderWithAlgo(hashAlgo); + + // The dump source may invoke the store, so we need to make some room. + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + conn.withFramedSink([&](Sink & sink) { dump.drainInto(sink); }); + } + + return make_ref(WorkerProto::Serialise::read(*this, *conn)); +} + void RemoteStore::queryRealisationUncached( const DrvOutput & id, Callback> callback) noexcept { diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 38ab658728bc..45d5ab9ffd6c 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -1,8 +1,10 @@ #include "nix/store/restricted-store.hh" #include "nix/store/build-result.hh" +#include "nix/store/submit-store.hh" #include "nix/util/callback.hh" #include "nix/store/realisation.hh" #include "nix/store/local-store.hh" +#include "nix/util/repair-flag.hh" namespace nix { @@ -38,7 +40,7 @@ bool RestrictionContext::isAllowed(const DerivedPath & req) * paths that are in the input closures of the build or were added via * recursive Nix calls. */ -struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore +struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore, public virtual SubmitStore { private: void anchor() override; @@ -112,6 +114,13 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void registerDrvOutput(const Realisation & info) override; + ref addToStoreScanning( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo) override; + void queryRealisationUncached( const DrvOutput & id, Callback> callback) noexcept override; @@ -256,6 +265,19 @@ void RestrictedStore::registerDrvOutput(const Realisation & info) throw Error("registerDrvOutput"); } +ref RestrictedStore::addToStoreScanning( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo) +{ + auto path = next->addToStoreFromDump( + dump, name, dumpMethod, hashMethod, hashAlgo, queryAllValidPaths(), RepairFlag::NoRepair, true); + + return queryPathInfo(path); +} + void RestrictedStore::queryRealisationUncached( const DrvOutput & id, Callback> callback) noexcept // XXX: This should probably be allowed if the realisation corresponds to diff --git a/src/libstore/submit-store.cc b/src/libstore/submit-store.cc new file mode 100644 index 000000000000..16e6feb848df --- /dev/null +++ b/src/libstore/submit-store.cc @@ -0,0 +1,7 @@ +#include "nix/store/submit-store.hh" + +namespace nix { + +void SubmitStore::anchor() {} + +} // namespace nix From c180cee5b10bab294647d62cb4f981b6b12087f8 Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Wed, 13 May 2026 14:52:18 -0400 Subject: [PATCH 347/364] nix store add: Allow scanning dependencies --- src/libstore/restricted-store.cc | 2 +- src/nix/add-to-store.cc | 41 +++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 45d5ab9ffd6c..a74addedaaf5 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -274,7 +274,7 @@ ref RestrictedStore::addToStoreScanning( { auto path = next->addToStoreFromDump( dump, name, dumpMethod, hashMethod, hashAlgo, queryAllValidPaths(), RepairFlag::NoRepair, true); - + goal.addDependency(path); return queryPathInfo(path); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 3c7939d1348f..fecbc242ef5e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,9 @@ #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" #include "nix/util/source-accessor.hh" +#include "nix/store/store-cast.hh" +#include "nix/store/submit-store.hh" +#include "nix/util/file-system.hh" #include "nix/cmd/misc-store-flags.hh" namespace nix { @@ -12,6 +15,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand std::optional namePart; ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; + bool scan = false; CmdAddToStore() { @@ -29,18 +33,49 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag(flag::contentAddressMethod(&caMethod)); addFlag(flag::hashAlgo(&hashAlgo)); + + addFlag({ + .longName = "scan", + .description = "Scan for references. Only works within a `recursive-nix` derivation builder.", + .handler = {&scan, true}, + }); } void run(ref store) override { + // Although this would be convenient, if we are scanning then we are connecting to a daemon. + // A dry-run scan would require either daemon-support for scanning a path for references + // or listing referenceable paths, both of which come with downsides. + if (dryRun && scan) + throw UsageError("Cannot dry-run while scanning"); + if (!namePart) namePart = path.filename().string(); auto sourcePath = makeFSSourceAccessor(absPath(path)); - auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first - : store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path; - + auto storePath = ([&]() { + if (scan) { + auto & submitStore = require(*store); + + auto serialisationMethod = caMethod.getFileSerialisationMethod(); + + std::optional storePath; + auto sink = sourceToSink([&](Source & source) { + auto info = + submitStore.addToStoreScanning(source, *namePart, serialisationMethod, caMethod, hashAlgo); + storePath = info->path; + }); + dumpPath(sourcePath, *sink, serialisationMethod, defaultPathFilter); + sink->finish(); + + return storePath.value(); + } else if (dryRun) { + return store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first; + } else { + return store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path; + } + })(); logger->cout("%s", store->printStorePath(storePath)); } }; From f5eea64623e0d59b46dd341e88af2282a9bcd920 Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Fri, 17 Jul 2026 10:40:39 -0400 Subject: [PATCH 348/364] tests: Add test for nix store add --scan --- tests/functional/add-scanning.nix | 38 +++++++++++++++++++++++++++++++ tests/functional/add-scanning.sh | 16 +++++++++++++ tests/functional/meson.build | 1 + 3 files changed, 55 insertions(+) create mode 100644 tests/functional/add-scanning.nix create mode 100755 tests/functional/add-scanning.sh diff --git a/tests/functional/add-scanning.nix b/tests/functional/add-scanning.nix new file mode 100644 index 000000000000..b5e8d13cdca4 --- /dev/null +++ b/tests/functional/add-scanning.nix @@ -0,0 +1,38 @@ +with import ./config.nix; + +let + dependency = mkDerivation { + name = "dependency"; + buildCommand = '' + mkdir $out + echo "this is a dependency" > $out/foo + ''; + }; +in +mkDerivation { + name = "add-scanning"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + set -euo pipefail + + PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH + export NIX_CONFIG='extra-experimental-features = nix-command' + + mkdir mao + echo "miao" > mao/foo + echo "${dependency}" > mao/reference + mao="$(nix store add --scan ./mao)" + + mkdir felis + echo "miau" > felis/foo + echo "$mao" > felis/reference + felis="$(nix store add --scan -n reference ./felis)" + + nix-store -qR "$felis" | grep "$mao" + nix-store -qR "$felis" | grep "${dependency}" + + nix-store -qR "$felis" > "$out" + ''; +} diff --git a/tests/functional/add-scanning.sh b/tests/functional/add-scanning.sh new file mode 100755 index 000000000000..04e06731031e --- /dev/null +++ b/tests/functional/add-scanning.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +requireDaemonNewerThan "2.36.0pre20260716" + +enableFeatures 'recursive-nix dynamic-derivations' +restartDaemon + +# grep is overridden with a function in this shell, is not in a new subshell +EXTRA_PATH=$(dirname "$(type -p nix)"):$(dirname "$(sh -c 'type -p grep')") +export EXTRA_PATH + +nix build -L --file ./add-scanning.nix --no-link diff --git a/tests/functional/meson.build b/tests/functional/meson.build index bb55e5474e3a..af1c7b9eaca7 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -53,6 +53,7 @@ suites = [ 'tests' : [ 'absolute-path-literals.sh', 'add.sh', + 'add-scanning.sh', 'bash-profile.sh', 'binary-cache-build-remote.sh', 'binary-cache-compression.sh', From a58925d4e41150e2ebde6e27579c556d9514e099 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Jul 2026 23:27:45 +0300 Subject: [PATCH 349/364] libutil: Use O_PATH when opening parent dirFds Also extend the symlinked-home functional NixOS VM test to catch such issues. --- src/libutil/posix-source-accessor.cc | 16 ++++++++++++++-- tests/nixos/functional/symlinked-home.nix | 14 ++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e426c43b34d3..55fa6342f451 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -383,8 +383,20 @@ std::pair> PosixDirectorySourceAccessor } try { - AutoCloseFD parentFdOwning = - openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); + AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( + startFd, + relPath, +# ifdef O_PATH + /* As to not require read permissions on the directory. */ + O_PATH | +# else + /* Sadly this will require read permissison for path resolution, + but without O_PATH that's unavoidable. */ + O_RDONLY | +# endif + O_DIRECTORY | O_CLOEXEC, + 0, + std::move(cb)); return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } catch (SymlinkNotAllowed & e) { /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ diff --git a/tests/nixos/functional/symlinked-home.nix b/tests/nixos/functional/symlinked-home.nix index 900543d0cfee..09780339da73 100644 --- a/tests/nixos/functional/symlinked-home.nix +++ b/tests/nixos/functional/symlinked-home.nix @@ -1,9 +1,11 @@ /** This test runs the functional tests on a NixOS system where the home directory - is symlinked to another location. + is symlinked to another location that also happens to reside in parent directory + that we don't have read permissions for (only execute). The purpose of this test is to find cases where Nix uses low-level operations - that don't support symlinks on paths that include them. + that don't support symlinks on paths that include them or requires excessive + permissions for path resolution. It is not a substitute for more intricate, use case-specific tests, but helps catch common issues. @@ -27,8 +29,12 @@ machine.succeed(""" ( set -x - mv /home/alice /home/alice.real - ln -s alice.real /home/alice + mkdir -p /home/alice.parent + chown alice:users /home/alice.parent + # Make the parent unreadable for good measure + chmod 0110 /home/alice.parent + mv /home/alice /home/alice.parent/alice.real + ln -s alice.parent/alice.real /home/alice ) 1>&2 """) machine.succeed(""" From adcea40393436cd052064e67d8fe8dc36021888a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Jul 2026 00:26:45 +0300 Subject: [PATCH 350/364] Use C23 embed instead of gen_headers This is now quite widely supported in Clang 19 and GCC 15: See: https://releases.llvm.org/19.1.0/tools/clang/docs/ReleaseNotes.html See: https://developers.redhat.com/articles/2025/01/30/how-implement-c23-embed-gcc-15 This change also caught a slight positioning bug, because we had UTF8 characters in derivation.nix and position tracking really doesn't work with multi-byte codepoints. It is caught because #embed apparently creates an initialiser list and it's a compile-time error to have narrowing conversions there. We can also simplify quite a lot of our cli documentation and #embed markdown instead of having weird raw-string-literal-in-a.md-file approach. --- nix-meson-build-support/common/meson.build | 2 ++ src/libexpr/eval.cc | 15 +++++---- src/libexpr/meson.build | 11 ------- src/libexpr/primops.cc | 2 +- src/libexpr/primops/derivation.nix | 4 +-- src/libexpr/primops/meson.build | 5 --- src/libflake/flake.cc | 5 +-- src/libflake/meson.build | 8 ----- .../linux/build/linux-derivation-builder.cc | 5 ++- src/libstore/local-store.cc | 15 +++++---- src/libstore/meson.build | 32 ++----------------- .../nix/util/memory-source-accessor.hh | 4 +++ src/nix/develop.cc | 8 ++--- src/nix/main.cc | 27 +++++++++------- src/nix/meson.build | 8 ++--- ...ion-structuredAttrs-stack-overflow.err.exp | 16 +++++----- 16 files changed, 64 insertions(+), 103 deletions(-) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index d4c89ce6f460..e9964bce2ac1 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -26,6 +26,8 @@ warning_flags = [ '-Werror=non-virtual-dtor', '-Wignored-qualifiers', '-Wimplicit-fallthrough', + # Clang complains about #embed even though it's now standard in C23. In C++ it's an extension, but meh. + '-Wno-c23-extensions', '-Wno-deprecated-declarations', '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. '-Wno-subobject-linkage', # GCC doesn't like unity builds. diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4bcb409707f7..7c92649b68e3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -296,12 +296,14 @@ EvalState::EvalState( , internalFS(make_ref()) , derivationInternal{internalFS->addFile( CanonPath("derivation-internal.nix"), -#include "primops/derivation.nix.gen.hh" - )} + { +#embed "primops/derivation.nix" + })} , importedDrvToDerivation{internalFS->addFile( CanonPath("imported-drv-to-derivation.nix"), -#include "imported-drv-to-derivation.nix.gen.hh" - )} + { +#embed "imported-drv-to-derivation.nix" + })} , store(store) , buildStore(buildStore ? buildStore : store) , inputCache(fetchers::InputCache::create()) @@ -363,8 +365,9 @@ EvalState::EvalState( corepkgsFS->addFile( CanonPath("fetchurl.nix"), -#include "fetchurl.nix.gen.hh" - ); + { +#embed "fetchurl.nix" + }); createBaseEnv(settings); diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index d44a0965d96b..6016fa81a79e 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -144,16 +144,6 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) -subdir('nix-meson-build-support/generate-header') - -generated_headers = [] -foreach header : [ - 'imported-drv-to-derivation.nix', - 'fetchurl.nix', -] - generated_headers += gen_header.process(header) -endforeach - sources = files( 'attr-path.cc', 'attr-set.cc', @@ -240,7 +230,6 @@ this_library = library( config_priv_h, parser_tab[1], lexer_tab[1], - generated_headers, soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 46ef49a36759..e93915c4b8fe 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -251,7 +251,7 @@ void derivationToValue( state.evalFile(state.importedDrvToDerivation, *vImportedDrvToDerivation); // has caching v.mkApp(vImportedDrvToDerivation, w); - state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix"); } /** diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index 1044fbd26db2..1d17c24ef11a 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,5 +1,5 @@ -# This is the implementation of the ‘derivation’ builtin function. -# It's actually a wrapper around the ‘derivationStrict’ primop. +# This is the implementation of the `derivation` builtin function. +# It's actually a wrapper around the `derivationStrict` primop. # Note that the following comment will be shown in :doc in the repl, but not in the manual. /** diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index b8abc6409af9..c49755970525 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -1,8 +1,3 @@ -generated_headers += gen_header.process( - 'derivation.nix', - preserve_path_from : meson.project_source_root(), -) - sources += files( 'context.cc', 'fetchClosure.cc', diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index bd3ea4acbc65..4f22353c751a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -916,8 +916,9 @@ static ref makeInternalFS() internalFS->setPathDisplay("«flakes-internal»", ""); internalFS->addFile( CanonPath("call-flake.nix"), -#include "call-flake.nix.gen.hh" // IWYU pragma: keep - ); + { +#embed "call-flake.nix" + }); return internalFS; } diff --git a/src/libflake/meson.build b/src/libflake/meson.build index c06bf6ba5450..c044bca7b46b 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -32,13 +32,6 @@ subdir('nix-meson-build-support/common') subdir('nix-meson-build-support/generate-header') -generated_headers = [] -foreach header : [ - 'call-flake.nix', -] - generated_headers += gen_header.process(header) -endforeach - sources = files( 'config.cc', 'flake-primops.cc', @@ -57,7 +50,6 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixflake', sources, - generated_headers, soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, diff --git a/src/libstore/linux/build/linux-derivation-builder.cc b/src/libstore/linux/build/linux-derivation-builder.cc index 310f796de854..525b3ffa1632 100644 --- a/src/libstore/linux/build/linux-derivation-builder.cc +++ b/src/libstore/linux/build/linux-derivation-builder.cc @@ -815,11 +815,10 @@ void ChrootLinuxDerivationBuilder::enterChroot() for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility - #if HAVE_EMBEDDED_SANDBOX_SHELL if (i.second.source == "__embedded_sandbox_shell__") { - static unsigned char sh[] = { -# include "embedded-sandbox-shell.gen.hh" + static constexpr unsigned char sh[] = { +# embed EMBEDDED_SANDBOX_SHELL_PATH }; auto dst = chrootRootDir / i.first.relative_path(); createDirs(dst.parent_path()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a1c161c0739..3e524703bf19 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -572,10 +572,9 @@ void LocalStore::openDB(State & state, bool create) /* Initialise the database schema, if necessary. */ if (create) { - static const char schema[] = -#include "schema.sql.gen.hh" - ; - db.exec(schema); + db.exec({ +#embed "schema.sql" + }); } } @@ -629,11 +628,13 @@ bool LocalStore::upgradeDBSchema(State & state, bool dryRun) schemaMigrations.insert(migrationName); }; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { maybeUpgrade( "20251017-ca-derivations", -#include "ca-specific-schema.sql.gen.hh" - ); + { +#embed "ca-specific-schema.sql" + }); + } maybeUpgrade("20260309-drop-redundant-indexreferrer", "drop index if exists IndexReferrer"); diff --git a/src/libstore/meson.build b/src/libstore/meson.build index c266f05d4dd1..a8827bb3fd18 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -176,40 +176,15 @@ endif configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int()) -subdir('nix-meson-build-support/generate-header') - -generated_headers = [] -foreach header : [ - 'schema.sql', - 'ca-specific-schema.sql', -] - generated_headers += gen_header.process(header) -endforeach - busybox = find_program(get_option('sandbox-shell'), required : false) -configdata_priv.set( - 'HAVE_EMBEDDED_SANDBOX_SHELL', - get_option('embedded-sandbox-shell').to_int(), -) - if get_option('embedded-sandbox-shell') configdata_priv.set_quoted('SANDBOX_SHELL', '__embedded_sandbox_shell__') + configdata_priv.set_quoted('EMBEDDED_SANDBOX_SHELL_PATH', busybox.full_path()) + configdata_priv.set('HAVE_EMBEDDED_SANDBOX_SHELL', 1) elif busybox.found() configdata_priv.set_quoted('SANDBOX_SHELL', busybox.full_path()) -endif - -if get_option('embedded-sandbox-shell') - hexdump = find_program('hexdump', native : true) - embedded_sandbox_shell_gen = custom_target( - 'embedded-sandbox-shell.gen.hh', - command : [ hexdump, '-v', '-e', '1/1 "0x%x," "\n"' ], - input : busybox.full_path(), - output : 'embedded-sandbox-shell.gen.hh', - capture : true, - feed : true, - ) - generated_headers += embedded_sandbox_shell_gen + configdata_priv.set('HAVE_EMBEDDED_SANDBOX_SHELL', 0) endif fs = import('fs') @@ -391,7 +366,6 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstore', - generated_headers, sources, config_priv_h, soversion : nix_soversion, diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index 6e1792abc3a3..7c7d38ab5fb5 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -144,6 +144,10 @@ public: */ File * open(const CanonPath & path, std::optional create); + /** + * @brief Insert a new regular file into with the specified @p contents. + * @todo Have a way to insert "borrowed" std::string_view without copying. + */ SourcePath addFile(CanonPath path, std::string && contents); }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1db0363be79c..ae115a6fab34 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -219,9 +219,9 @@ struct BuildEnvironment } }; -const static std::string getEnvSh = -#include "get-env.sh.gen.hh" - ; +static constexpr char getEnvSh[] = { +#embed "get-env.sh" +}; /** * Given an existing derivation, return the shell environment as @@ -239,7 +239,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); auto getEnvShPath = ({ - StringSource source{getEnvSh}; + StringSource source{std::string_view(getEnvSh, sizeof(getEnvSh))}; evalStore->addToStoreFromDump( source, "get-env.sh", diff --git a/src/nix/main.cc b/src/nix/main.cc index 963bc07ce274..ee3fe27c48ef 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -257,24 +257,29 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval( state.parseExprFromString( -#include "generate-manpage.nix.gen.hh" - , state.rootPath(CanonPath::root)), + { +#embed "doc/manual/generate-manpage.nix" + }, + state.rootPath(CanonPath::root)), *vGenerateManpage); state.corepkgsFS->addFile( CanonPath("utils.nix"), -#include "utils.nix.gen.hh" - ); + { +#embed "doc/manual/utils.nix" + }); state.corepkgsFS->addFile( CanonPath("/generate-settings.nix"), -#include "generate-settings.nix.gen.hh" - ); + { +#embed "doc/manual/generate-settings.nix" + }); state.corepkgsFS->addFile( CanonPath("/generate-store-info.nix"), -#include "generate-store-info.nix.gen.hh" - ); + { +#embed "doc/manual/generate-store-info.nix" + }); auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli(), state.mem); @@ -348,9 +353,9 @@ struct CmdHelpStores : Command std::string doc() override { - return -#include "help-stores.md.gen.hh" - ; + return { +#embed "help-stores.md" + }; } Category category() override diff --git a/src/nix/meson.build b/src/nix/meson.build index 346f7e4a5c85..45d584335702 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -134,13 +134,9 @@ if host_machine.system() != 'windows' endif nix_sources += [ - gen_header.process('doc/manual/generate-manpage.nix'), - gen_header.process('doc/manual/generate-settings.nix'), - gen_header.process('doc/manual/generate-store-info.nix'), - gen_header.process('doc/manual/utils.nix'), - gen_header.process('get-env.sh'), + # TODO: Get rid of this for good. It's used in profile.md with + # string literal concatenation. gen_header.process('profiles.md'), - gen_header.process('help-stores.md'), ] # The rest of the subdirectories aren't separate components, diff --git a/tests/functional/lang/eval-fail-derivation-structuredAttrs-stack-overflow.err.exp b/tests/functional/lang/eval-fail-derivation-structuredAttrs-stack-overflow.err.exp index ec849e15d1fe..91b7f11e1489 100644 --- a/tests/functional/lang/eval-fail-derivation-structuredAttrs-stack-overflow.err.exp +++ b/tests/functional/lang/eval-fail-derivation-structuredAttrs-stack-overflow.err.exp @@ -1,17 +1,17 @@ error: … while evaluating the attribute 'outPath' - at «nix-internal»/derivation-internal.nix:50:7: - 49| value = commonAttrs // { - 50| outPath = strict.${outputName}; + at «nix-internal»/derivation-internal.nix:49:7: + 48| value = commonAttrs // { + 49| outPath = strict.${outputName}; | ^ - 51| drvPath = strict.drvPath; + 50| drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at «nix-internal»/derivation-internal.nix:37:12: - 36| - 37| strict = derivationStrict drvAttrs; + at «nix-internal»/derivation-internal.nix:36:12: + 35| + 36| strict = derivationStrict drvAttrs; | ^ - 38| + 37| … while evaluating derivation 'test' whose name attribute is located at /pwd/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix:5:3 From ebef6b2193631c8944d46f75ea37e3cb8b2de663 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 20 Jul 2026 12:51:26 -0500 Subject: [PATCH 351/364] Fix crash in restricted store queryRealisationUncached Ensure the callback is only called once. Signed-off-by: Lisanna Dettwyler --- src/libstore/restricted-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index a74addedaaf5..9935f1f972ec 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -283,8 +283,10 @@ void RestrictedStore::queryRealisationUncached( // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation { - if (!goal.isAllowed(id)) + if (!goal.isAllowed(id)) { callback(nullptr); + return; + } next->queryRealisation(id, std::move(callback)); } From 4125ece6c1ad8d2809f30431f00d379dd014c731 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 20 Mar 2026 17:04:57 -0400 Subject: [PATCH 352/364] Separate building/scheduling from storage This is the major first step of #5025. Motivation ---------- Right now, there is a bit of conceptual tension between `--store` and `--builders`: - With `--store`, it is very convenient to think that the store knows how to build. One specifies a store, and gets a different method of building (local, some sort of remote) and scheduling (the remote store can take an entire derivation graph, multiple jobs) accordingly. - With `--builders` one has a local scheduler. Stores either act as a passive "workbench" for building (the local case) or we give them a single job (a single ready-to-build derivation) at a time. For historical reasons, this doesn't even use the store interface, except on the "other side" of the build hook. Issue #5025 is about using the store interface for `--builders`. In this case, we want to invert the relationship between `Store` and `Worker`. We have no use in this case for various `Store` methods creating a `Worker` behind the scenes, because we already have our `Worker`: - `LocalStore`s don't need any build method at all, we can just have our `Worker` directly use the local store, as it does with the default `worker.store` today. - remote stores supporting building (`ssh://` and `ssh-ng://`) are only fed a single job at a time, their remote-side scheduling being overkill for the task at hand. But we can't just delete the building methods of `--store` that we don't need anymore, because that would break `--store` building. We need to support both cases, where some stores effectively build/schedule and `Worker` can also own/borrow stores to be a single, unified scheduler. This change ----------- The way we satisfy both goals is by: - Pulling the building methods out of `Store` into a new `Builder` class - Having some stores also give/implement `Builder` The separation of `Store` vs `Builder` works for the `--builders` use-case, and the project of making that leverage `Store` and other C++ interfaces directly without indirecting through build hook or other ad-hoc implementation swapping methods. Here's how: as opposed to default `Store::` method implementations creating a `Worker` on the fly, `Worker` will implement `Builder`, and those methods, now on `Builder`, will become `Worker`'s own implementation. Local building To implement this conceptual switch, the methods that directly delegated to the worker are now instead ripped off `Store` and put in the new `Builder` class. (For example, `build/entry-points.cc` now contains all `Worker` methods (virtual method impls of `Builder`) and not `Store` method impls.) (`Worker` should be renamed to `LocalBuilder`, since it is the local build scheduler, and additionally knows how to build in local stores.) Remote building What about the `--store` case? The remote stores have a new method to provide a `Builder` of their choice given an `evalStore`. (This reflects the fact that `Builder` no longer has `evalStore` parameters on its methods.) That new method is a new interface `BuildStore` which `RemoteStore` and `LegacySSHStore` implement. Each one has an unexposed `Builder` implementation which will just do everything over RPC, like today. Putting it all together Introduce: - `Store::getBuilder`, which returns an owning reference to something implementing Builder. - `LocalBuilder`, a wrapper around `Worker` to enable thread-safety and optimized ensurePath. Future work ----------- Issue NixOS#1221 The next step of the #5025 saga is issue #1221. To solve that issue, `Worker` will not use the build hook, but instead work via C++. In particular, it will do this: - if the builder is a `BuildStore`, use an appropriate method (possibly yet to be created) on the remote building store's (remote) `Builder`. - if the builder is a `LocalStore` `Worker` should *not* create another `Worker` (as the `build-remote` program would do today) but instead directly manage building in that local store, so we avoid **n** `Worker` instances scheduling independently (which is stupid discoordination). - (Otherwise fail, which matches what happens today, actually, just in fewer steps.) Simplifying the RPC case I also suspect that longer term, those stores will just implement `Builder` directly, as `evalStore` doesn't really make sense for RPC endpoints when the remote side has no idea what the caller is doing with other stores. We won't need `BuildStore` anymore then. Other improvements ------------------ Recursive Nix Speaking of avoiding redundant schedulers: `RestrictedStore`, when it used to override the store building methods, would spin up a new `Worker` for each recursive Nix build call. This is again bad --- we should have a central scheduler that takes in dynamic jobs, same for recursive Nix and dynamic derivations. Now this is *almost*, but not quite, fixed. Three changes were made: - `RestrictedBuilder` was split out from `RestrictedStore` to wrap the build methods. - `processConnection` takes an optional `Builder` parameter, using it directly rather than spinning one up with `getDefaultBuilder`. - `DerivationBuilder` took a callback to process the connection for recursive Nix, so the caller could provide the `processConnection` call with the ambient worker in order to reuse it. This would have solved the redundant scheduler problem very nicely! This unfortunately deadlocked, so instead the caller explicitly creates a fresh worker (as before, but not hidden beneath a gazillion abstractions) with a TODO saying the deadlock should be fixed and this should not be done. `LegacySSHStore` fix As a final note, the old `LegacySSHStore` did not override `buildPathsWithResults`, which meant that when specifying an `ssh://` store, the local scheduler was being erroneously used for some commands. Now, `LegacySSHBuilder::buildPathsWithResults` uses a single `buildPathsRaw` call (which sends the serve protocol `BuildPaths` command and returns `std::variant` with the error message already read from the wire), and then queries realisations to reconstruct the `BuildResult`s --- code similar to the old fallback code for `ssh-ng://`. Use std::shared_ptr for processConnection's builder This avoids the need to pass a raw pointer. Signed-off-by: Lisanna Dettwyler Rename BuildStore to BuildStore Signed-off-by: Lisanna Dettwyler Co-authored-by: John Ericson --- src/libcmd/installables.cc | 3 +- src/libcmd/repl.cc | 3 +- src/libexpr/primops.cc | 5 +- src/libexpr/primops/context.cc | 3 +- src/libexpr/primops/fetchTree.cc | 3 +- src/libfetchers/fetchers.cc | 3 +- src/libstore-c/nix_api_store.cc | 3 +- .../build/derivation-building-goal.cc | 18 ++ src/libstore/build/entry-points.cc | 91 +++++---- src/libstore/daemon.cc | 25 ++- src/libstore/include/nix/store/build.hh | 97 ++++++++++ .../nix/store/build/derivation-builder.hh | 7 + .../include/nix/store/build/worker.hh | 47 ++++- src/libstore/include/nix/store/daemon.hh | 18 +- .../include/nix/store/legacy-ssh-store.hh | 35 +--- src/libstore/include/nix/store/machines.hh | 7 + src/libstore/include/nix/store/meson.build | 1 + .../include/nix/store/remote-store.hh | 25 +-- .../include/nix/store/restricted-store.hh | 8 + src/libstore/include/nix/store/store-api.hh | 92 ++------- src/libstore/legacy-ssh-store.cc | 183 +++++++++++++++--- src/libstore/local-store.cc | 5 +- src/libstore/misc.cc | 27 --- src/libstore/remote-store.cc | 105 +++++++--- src/libstore/restricted-store.cc | 98 ++++++---- src/libstore/store-api.cc | 15 +- src/libstore/unix/build/derivation-builder.cc | 6 +- src/nix/build-remote/build-remote.cc | 5 +- src/nix/bundle.cc | 3 +- src/nix/develop.cc | 6 +- src/nix/flake.cc | 4 +- src/nix/nix-build/nix-build.cc | 3 +- src/nix/nix-env/nix-env.cc | 3 +- src/nix/nix-env/user-env.cc | 7 +- src/nix/nix-store/nix-store.cc | 13 +- src/nix/store-repair.cc | 3 +- src/nix/upgrade-nix.cc | 3 +- .../functional/test-libstoreconsumer/main.cc | 3 +- 38 files changed, 660 insertions(+), 326 deletions(-) create mode 100644 src/libstore/include/nix/store/build.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 82edbfca0ee2..6c0de1d8966a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -15,6 +15,7 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" @@ -631,7 +632,7 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + auto buildResults = store->getBuilder(evalStore)->buildPathsWithResults(pathsToBuild, bMode); throwBuildErrors(buildResults, *store); for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index c6523fe57389..ed35988368d8 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -26,6 +26,7 @@ #include "nix/util/finally.hh" #include "nix/cmd/markdown.hh" #include "nix/store/local-fs-store.hh" +#include "nix/store/build.hh" #include "nix/expr/print.hh" #include "nix/util/ref.hh" #include "nix/expr/value.hh" @@ -548,7 +549,7 @@ ProcessLineResult NixRepl::processLine(std::string line) std::string drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b" || command == ":bl") { - state->store->buildPaths({ + state->store->getBuilder()->buildPaths({ DerivedPath::Built{ .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 46ef49a36759..129134fa9276 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -11,6 +11,7 @@ #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" #include "nix/util/mounted-source-accessor.hh" +#include "nix/store/build.hh" #include "nix/util/util.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" @@ -126,7 +127,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS buildReqs.reserve(drvs.size()); for (auto & d : drvs) buildReqs.emplace_back(DerivedPath{d}); - buildStore->buildPaths(buildReqs, bmNormal, store); + buildStore->getBuilder(store)->buildPaths(buildReqs, bmNormal); StorePathSet outputsToCopyAndAllow; @@ -1973,7 +1974,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) - state.store->ensurePath(storePath); + state.store->getBuilder()->ensurePath(storePath); context.insert(NixStringContextElem::Opaque{.path = storePath}); v.mkString(sourcePath.path.abs(), context, state.mem); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index d5d5de0b9aaf..7eef9daa2560 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -2,6 +2,7 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/globals.hh" namespace nix { @@ -272,7 +273,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg state.error("context key '%s' is not a store path", name).atPos(i.pos).debugThrow(); auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(namePath); + state.store->getBuilder()->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); if (auto attr = i.value->attrs()->get(sPath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0e092927b39d..6ffcf91ed30a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -5,6 +5,7 @@ #include "nix/expr/eval-settings.hh" #include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" #include "nix/fetchers/registry.hh" @@ -562,7 +563,7 @@ static void fetch( // Try to get the path from the local store or substituters try { - state.store->ensurePath(expectedPath); + state.store->getBuilder()->ensurePath(expectedPath); debug("using substituted/cached path '%s' for '%s'", state.store->printStorePath(expectedPath), *url); state.allowAndSetStorePathString(expectedPath, v); return; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 53bcf12dbe69..02132b381463 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" #include "nix/util/fs-sink.hh" +#include "nix/store/build.hh" #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" @@ -321,7 +322,7 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings store.addTempRoot(storePath); - store.ensurePath(storePath); + store.getBuilder()->ensurePath(storePath); debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath)); diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fbb3c418566c..bce1cdd22b22 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -8,6 +8,7 @@ #include "nix/store/path.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/store-open.hh" #include "nix/store/store-reference.hh" #include "nix/store/build-result.hh" @@ -178,7 +179,7 @@ nix_err nix_store_realise( .drvPath = nix::makeConstantStorePathRef(path->path), .outputs = nix::OutputsSpec::All{}}}; const auto nixStore = store->ptr; - auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore); + auto results = nixStore->getBuilder(nixStore)->buildPathsWithResults(paths, nix::bmNormal); assert(results.size() == 1); diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 5531f984b966..386f591d842b 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -1,5 +1,7 @@ #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-env-desugar.hh" +#include "nix/store/restricted-store.hh" +#include "nix/store/daemon.hh" #ifndef _WIN32 // TODO enable build hook on Windows # include "nix/store/build/hook-instance.hh" # include "nix/store/build/derivation-builder.hh" @@ -940,6 +942,22 @@ Goal::Co DerivationBuildingGoal::buildLocally( { closeLogFileFn(); } + + void processDaemonConnection( + ref store, FdSource && from, FdSink && to, RestrictionContext & context) override + { + /** + * TODO: We create a fresh Worker here because the + * parent Worker is blocked waiting for the current + * build to finish, so we can't reuse it from a + * daemon thread. Ideally we should reuse the same + * Worker to share scheduling state. + */ + Worker freshWorker{goal.worker.store, goal.worker.evalStore}; + auto builder = makeRestrictedBuilder(freshWorker, context); + daemon::processConnection( + store, std::move(from), std::move(to), NotTrusted, daemon::Recursive, builder.get_ptr()); + } }; decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 5b97966847b8..e4fceb364a85 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -3,18 +3,47 @@ #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" #include "nix/util/strings.hh" +#include namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) +void LocalBuilder::buildPaths(const std::vector & reqs, BuildMode buildMode) { - Worker worker(*this, evalStore ? *evalStore : *this); + getWorker()->buildPaths(reqs, buildMode); +} + +std::vector +LocalBuilder::buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) +{ + return getWorker()->buildPathsWithResults(reqs, buildMode); +} + +BuildResult LocalBuilder::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +{ + return getWorker()->buildDerivation(drvPath, drv, buildMode); +} + +void LocalBuilder::ensurePath(const StorePath & path) +{ + /* If the path is already valid, we're done. */ + if (store->isValidPath(path)) + return; + getWorker()->ensurePath(path); +} + +void LocalBuilder::repairPath(const StorePath & path) +{ + getWorker()->repairPath(path); +} + +void Worker::buildPaths(const std::vector & reqs, BuildMode buildMode) +{ Goals goals; for (auto & br : reqs) - goals.insert(worker.makeGoal(br, buildMode)); + goals.insert(makeGoal(br, buildMode)); - worker.run(goals); + run(goals); StringSet failed; BuildResult::Failure * failure = nullptr; @@ -27,38 +56,35 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvReq->to_string(*this)); + failed.insert(i2->drvReq->to_string(store)); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(printStorePath(i2->storePath)); + failed.insert(store.printStorePath(i2->storePath)); } } if (failed.size() == 1 && failure) { - failure->withExitStatus(worker.exitStatusFlags.failingExitStatus()); + failure->withExitStatus(exitStatusFlags.failingExitStatus()); throw *failure; } else if (!failed.empty()) { - auto exitStatus = worker.exitStatusFlags.failingExitStatus(); + auto exitStatus = exitStatusFlags.failingExitStatus(); if (failure) logError(failure->info()); throw Error(exitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } -std::vector Store::buildPathsWithResults( - const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) +std::vector Worker::buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) { - Worker worker(*this, evalStore ? *evalStore : *this); - Goals goals; std::vector> state; for (const auto & req : reqs) { - auto goal = worker.makeGoal(req, buildMode); + auto goal = makeGoal(req, buildMode); goals.insert(goal); state.push_back({req, goal}); } - worker.run(goals); + run(goals); std::vector results; results.reserve(state.size()); @@ -79,13 +105,12 @@ std::vector Store::buildPathsWithResults( return results; } -BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +BuildResult Worker::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - Worker worker(*this, *this); - auto goal = worker.makeDerivationTrampolineGoal(drvPath, OutputsSpec::All{}, drv, buildMode); + auto goal = makeDerivationTrampolineGoal(drvPath, OutputsSpec::All{}, drv, buildMode); try { - worker.run(Goals{goal}); + run(Goals{goal}); return goal->buildResult; } catch (Error & e) { return BuildResult{ @@ -96,49 +121,47 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat }; } -void Store::ensurePath(const StorePath & path) +void Worker::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ - if (isValidPath(path)) + if (store.isValidPath(path)) return; - Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path); + GoalPtr goal = makePathSubstitutionGoal(path); Goals goals = {goal}; - worker.run(goals); + run(goals); if (goal->exitCode != Goal::ecSuccess) { - auto exitStatus = worker.exitStatusFlags.failingExitStatus(); + auto exitStatus = exitStatusFlags.failingExitStatus(); goal->buildResult.tryThrowBuildError(exitStatus); - throw Error(exitStatus, "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(exitStatus, "path '%s' does not exist and cannot be created", store.printStorePath(path)); } } -void Store::repairPath(const StorePath & path) +void Worker::repairPath(const StorePath & path) { - Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); + GoalPtr goal = makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; - worker.run(goals); + run(goals); if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { + auto info = store.queryPathInfo(path); + if (info->deriver && store.isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeGoal( + goals.insert(makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. .outputs = OutputsSpec::All{}, }, bmRepair)); - worker.run(goals); + run(goals); } else - throw Error(worker.exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", store.printStorePath(path)); } } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index bac95818ff46..23df48bfc29d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -4,6 +4,7 @@ #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" +#include "nix/store/build.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/filetransfer.hh" @@ -308,7 +309,8 @@ static void performOp( TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::BasicServerConnection & conn, - WorkerProto::Op op) + WorkerProto::Op op, + Builder & builder) { WorkerProto::ReadConn rconn(conn); WorkerProto::WriteConn wconn(conn); @@ -555,7 +557,7 @@ static void performOp( if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); - store->buildPaths(drvs, mode); + builder.buildPaths(drvs, mode); logger->stopWork(); conn.to << 1; break; @@ -574,7 +576,7 @@ static void performOp( throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); - auto results = store->buildPathsWithResults(drvs, mode); + auto results = builder.buildPathsWithResults(drvs, mode); logger->stopWork(); WorkerProto::write(*store, wconn, results); @@ -653,7 +655,7 @@ static void performOp( drvPath = store->writeDerivation(Derivation{drv2}); } - auto res = store->buildDerivation(drvPath, drv, buildMode); + auto res = builder.buildDerivation(drvPath, drv, buildMode); logger->stopWork(); WorkerProto::write(*store, wconn, res); break; @@ -662,7 +664,7 @@ static void performOp( case WorkerProto::Op::EnsurePath: { auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); - store->ensurePath(path); + builder.ensurePath(path); logger->stopWork(); conn.to << 1; break; @@ -1052,7 +1054,13 @@ static void performOp( } } -void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) +void processConnection( + ref store, + FdSource && from, + FdSink && to, + TrustedFlag trusted, + RecursiveFlag recursive, + std::shared_ptr builder) { #ifndef _WIN32 // TODO need graceful async exit support on Windows? auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -1070,6 +1078,9 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted }); #endif + if (!builder) + builder = store->getBuilder(); + /* Exchange the greeting. */ auto localVersion = WorkerProto::latest; if (recursive) { @@ -1136,7 +1147,7 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, conn, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op, *builder); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the diff --git a/src/libstore/include/nix/store/build.hh b/src/libstore/include/nix/store/build.hh new file mode 100644 index 000000000000..dbf2fc76ce7b --- /dev/null +++ b/src/libstore/include/nix/store/build.hh @@ -0,0 +1,97 @@ +#pragma once +///@file + +#include "nix/store/store-api.hh" + +namespace nix { + +/** + * Abstract interface for the build scheduler entry points. + * + * `Worker` implements this for local scheduling, including local builds. + * Remote stores provide a `Builder` via `Store::getBuilder()`. + * + * Thread safety should be guaranteed across these methods. + */ +struct Builder +{ + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ + virtual void buildPaths(const std::vector & reqs, BuildMode buildMode = bmNormal) = 0; + + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ + virtual std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode = bmNormal) = 0; + + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ + virtual BuildResult + buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; + + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ + virtual void ensurePath(const StorePath & path) = 0; + + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ + virtual void repairPath(const StorePath & path) = 0; + + virtual ~Builder() = default; +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index ea2e46445142..08fce39b2777 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -140,6 +140,13 @@ struct DerivationBuilderCallbacks * @todo this should be reworked */ virtual void childTerminated() = 0; + + /** + * Process a recursive Nix daemon connection, using a builder + * that enforces the restrictions of the given context. + */ + virtual void + processDaemonConnection(ref store, FdSource && from, FdSink && to, RestrictionContext & context) = 0; }; /** diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 12e6c0122e1c..d8bbfa1ef5c4 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -3,6 +3,7 @@ #include "nix/util/types.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/derived-path-map.hh" #include "nix/store/build/goal.hh" #include "nix/store/build-result.hh" @@ -67,10 +68,44 @@ struct Child struct HookInstance; #endif +/** + * Owns a worker. Optimization around ensurePath to prevent a Worker from + * being constructed when it's not needed. + */ +class LocalBuilder : public Builder +{ +public: + LocalBuilder(ref store, ref evalStore) + : store(store) + , evalStore(evalStore) {}; + + /* Builder interface — see `Builder` for documentation. */ + + void buildPaths(const std::vector & reqs, BuildMode buildMode) override; + std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + void ensurePath(const StorePath & path) override; + void repairPath(const StorePath & path) override; + +private: + /** + * Intentionally construct a new worker for each operation, to avoid + * reusing a worker between calls, allowing for thread safety. + */ + inline std::shared_ptr getWorker() + { + return std::make_shared(*store, *evalStore); + } + + ref store; + ref evalStore; +}; + /** * Coordinates one or more realisations and their interdependencies. */ -class Worker +class Worker : public Builder { private: @@ -199,6 +234,7 @@ public: Store & store; Store & evalStore; + const WorkerSettings & settings; /** @@ -388,6 +424,15 @@ public: act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } + + /* Builder interface — see `Builder` for documentation. */ + + void buildPaths(const std::vector & reqs, BuildMode buildMode) override; + std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + void ensurePath(const StorePath & path) override; + void repairPath(const StorePath & path) override; }; } // namespace nix diff --git a/src/libstore/include/nix/store/daemon.hh b/src/libstore/include/nix/store/daemon.hh index 4d550696e877..a01205a976fd 100644 --- a/src/libstore/include/nix/store/daemon.hh +++ b/src/libstore/include/nix/store/daemon.hh @@ -4,10 +4,22 @@ #include "nix/util/serialise.hh" #include "nix/store/store-api.hh" -namespace nix::daemon { +namespace nix { + +struct Builder; + +namespace daemon { enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; -void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); +void processConnection( + ref store, + FdSource && from, + FdSink && to, + TrustedFlag trusted, + RecursiveFlag recursive, + std::shared_ptr builder = nullptr); + +} // namespace daemon -} // namespace nix::daemon +} // namespace nix diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 839cc8df74b2..7efbdf1545df 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -2,7 +2,6 @@ ///@file #include "nix/store/common-ssh-store-config.hh" -#include "nix/store/store-api.hh" #include "nix/store/ssh.hh" #include "nix/util/callback.hh" #include "nix/util/pool.hh" @@ -140,24 +139,7 @@ public: public: - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - - /** - * Note, the returned function must only be called once, or we'll - * try to read from the connection twice. - * - * @todo Use C++23 `std::move_only_function`. - */ - fun buildDerivationAsync( - const StorePath & drvPath, const BasicDerivation & drv, const ServeProto::BuildOptions & options); - - void buildPaths( - const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; - - void ensurePath(const StorePath & path) override - { - unsupported("ensurePath"); - } + ref getBuilder(std::shared_ptr evalStore) override; ref getFSAccessor(bool requireValidPath) override { @@ -169,19 +151,6 @@ public: unsupported("getFSAccessor"); } - /** - * The default instance would schedule the work on the client side, but - * for consistency with `buildPaths` and `buildDerivation` it should happen - * on the remote side. - * - * We make this fail for now so we can add implement this properly later - * without it being a breaking change. - */ - void repairPath(const StorePath & path) override - { - unsupported("repairPath"); - } - void computeFSClosure( const StorePathSet & paths, StorePathSet & out, @@ -232,6 +201,8 @@ public: // not supported return {}; } + + friend struct LegacySSHBuilder; }; } // namespace nix diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index a3e9353c80fa..50d5350838b0 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -16,6 +16,13 @@ struct Machine { const StoreReference storeUri; + /** + * @TODO this information should eventually just exist to update an + * underlying setting on `Store::Config`, just as the feature information + * updates `Store::Config::systemType`. The only wrinkle is whether the + * makes sense for separate local stores to have distinct systems, when they + * are all the current OS, just different part of the file system. + */ const StringSet systemTypes; const std::optional sshKey; const unsigned int maxJobs; diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 5a3d21e106ea..fb4f0658b47a 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -13,6 +13,7 @@ headers = [ config_pub_h ] + files( 'aws-creds.hh', 'binary-cache-store.hh', 'build-result.hh', + 'build.hh', 'build/build-log.hh', 'build/derivation-builder.hh', 'build/derivation-building-goal.hh', diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 14e8af3b00dd..295f37ab0313 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -124,15 +124,7 @@ public: void queryRealisationUncached( const DrvOutput &, Callback> callback) noexcept override; - void - buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - - std::vector buildPathsWithResults( - const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - - void ensurePath(const StorePath & path) override; + ref getBuilder(std::shared_ptr evalStore) override; void addTempRoot(const StorePath & path) override; @@ -150,19 +142,6 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; - /** - * The default instance would schedule the work on the client side, but - * for consistency with `buildPaths` and `buildDerivation` it should happen - * on the remote side. - * - * We make this fail for now so we can add implement this properly later - * without it being a breaking change. - */ - void repairPath(const StorePath & path) override - { - unsupported("repairPath"); - } - void addSignatures(const StorePath & storePath, const std::set & sigs) override; MissingPaths queryMissing(const std::vector & targets) override; @@ -230,7 +209,7 @@ private: */ Sync> connectionFds; - void copyDrvsFromEvalStore(const std::vector & paths, std::shared_ptr evalStore); + friend struct RemoteBuilder; }; } // namespace nix diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index aad0a9695104..5b6e8b734c8e 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -8,8 +8,10 @@ namespace nix { +struct Builder; class LocalStore; struct LocalStoreConfig; +class Worker; /** * A restricted store has a pointer to one of these, which manages the @@ -116,4 +118,10 @@ protected: */ ref makeRestrictedStore(ref config, ref next, RestrictionContext & context); +/** + * Create a builder that wraps an inner builder, adding restriction + * checks and dependency tracking for recursive Nix builds. + */ +ref makeRestrictedBuilder(Worker & inner, RestrictionContext & context); + } // namespace nix diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 68a18f48db67..1e77ecfe5ccf 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -57,6 +57,8 @@ enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; struct KeyedBuildResult; +struct Builder; + typedef std::map> StorePathCAMap; /** @@ -455,6 +457,15 @@ public: virtual ~Store() {} + /** + * Get a `Builder` for this store. + * + * @param evalStore If provided and different from this store, + * derivation files will be copied from the eval store to this + * store before building. + */ + virtual ref getBuilder(std::shared_ptr evalStore = nullptr); + /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -481,6 +492,8 @@ public: * If requested, substitute missing paths. This * implements nix-copy-closure's --use-substitutes * flag. + * + * @TODO suspicious to have a Store method that uses `getBuilder`. */ void substitutePaths(const StorePathSet & paths); @@ -743,77 +756,6 @@ public: */ virtual void narFromPath(const StorePath & path, Sink & sink); - /** - * For each path, if it's a derivation, build it. Building a - * derivation means ensuring that the output paths are valid. If - * they are already valid, this is a no-op. Otherwise, validity - * can be reached in two ways. First, if the output paths is - * substitutable, then build the path that way. Second, the - * output paths can be created by running the builder, after - * recursively building any sub-derivations. For inputs that are - * not derivations, substitute them. - */ - virtual void buildPaths( - const std::vector & paths, - BuildMode buildMode = bmNormal, - std::shared_ptr evalStore = nullptr); - - /** - * Like buildPaths(), but return a vector of \ref BuildResult - * BuildResults corresponding to each element in paths. Note that in - * case of a build/substitution error, this function won't throw an - * exception, but return a BuildResult containing an error message. - */ - virtual std::vector buildPathsWithResults( - const std::vector & paths, - BuildMode buildMode = bmNormal, - std::shared_ptr evalStore = nullptr); - - /** - * Build a single non-materialized derivation (i.e. not from an - * on-disk .drv file). - * - * @param drvPath This is used to deduplicate worker goals so it is - * imperative that is correct. That said, it doesn't literally need - * to be store path that would be calculated from writing this - * derivation to the store: it is OK if it instead is that of a - * Derivation which would resolve to this (by taking the outputs of - * it's input derivations and adding them as input sources) such - * that the build time referenceable-paths are the same. - * - * In the input-addressed case, we usually *do* use an "original" - * unresolved derivations's path, as that is what will be used in the - * buildPaths case. Also, the input-addressed output paths are verified - * only by that contents of that specific unresolved derivation, so it is - * nice to keep that information around so if the original derivation is - * ever obtained later, it can be verified whether the trusted user in fact - * used the proper output path. - * - * In the content-addressed case, we want to always use the resolved - * drv path calculated from the provided derivation. This serves two - * purposes: - * - * - It keeps the operation trustless, by ruling out a maliciously - * invalid drv path corresponding to a non-resolution-equivalent - * derivation. - * - * - For the floating case in particular, it ensures that the derivation - * to output mapping respects the resolution equivalence relation, so - * one cannot choose different resolution-equivalent derivations to - * subvert dependency coherence (i.e. the property that one doesn't end - * up with multiple different versions of dependencies without - * explicitly choosing to allow it). - */ - virtual BuildResult - buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - - /** - * Ensure that a path is valid. If it is not currently valid, it - * may be made valid by running a substitute (if defined for the - * path). - */ - virtual void ensurePath(const StorePath & path); - /** * Add a store path as a temporary root of the garbage collector. * The root disappears as soon as we exit. @@ -920,12 +862,6 @@ public: return ref{accessor}; } - /** - * Repair the contents of the given path by redownloading it using - * a substituter (if available). - */ - virtual void repairPath(const StorePath & path); - /** * Add signatures to the specified store path. The signatures are * not verified. @@ -948,6 +884,8 @@ public: /** * Read a derivation, after ensuring its existence through * ensurePath(). + * + * @TODO suspicious to have a Store method that uses `getBuilder`. */ Derivation derivationFromPath(const StorePath & drvPath); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index cbcc42dbed75..1765c5af747a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,12 +12,61 @@ #include "nix/store/path-with-outputs.hh" #include "nix/store/ssh.hh" #include "nix/store/derivations.hh" +#include "nix/store/build.hh" #include "nix/util/callback.hh" #include "nix/store/store-registration.hh" #include "nix/store/globals.hh" namespace nix { +struct LegacySSHBuilder : Builder +{ + ref store; + + LegacySSHBuilder(ref store) + : store(store) + { + } + +private: + + [[noreturn]] void unsupported(const std::string & op) + { + throw Unsupported("operation '%s' is not supported by store '%s'", op, store->config->getHumanReadableURI()); + } + + std::variant + buildPathsRaw(const std::vector & reqs, BuildMode buildMode); + +public: + + void buildPaths(const std::vector & reqs, BuildMode buildMode) override; + + std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + + /** + * Note, the returned function must only be called once, or we'll + * try to read from the connection twice. + * + * @todo Use C++23 `std::move_only_function`. + */ + fun buildDerivationAsync( + const StorePath & drvPath, const BasicDerivation & drv, const ServeProto::BuildOptions & options); + + void ensurePath(const StorePath & path) override + { + unsupported("ensurePath"); + } + + void repairPath(const StorePath & path) override + { + unsupported("repairPath"); + } +}; + LegacySSHStoreConfig::LegacySSHStoreConfig(const ParsedURL::Authority & authority, const Params & params) : StoreConfig(params, FilePathType::Unix) , CommonSSHStoreConfig(authority, params) @@ -196,32 +245,30 @@ static ServeProto::BuildOptions buildSettings() }; } -BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +BuildResult +LegacySSHBuilder::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(store->connections->get()); - conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings()); + conn->putBuildDerivationRequest(*store, drvPath, drv, buildSettings()); - return conn->getBuildDerivationResponse(*this); + return conn->getBuildDerivationResponse(*store); } -fun LegacySSHStore::buildDerivationAsync( +fun LegacySSHBuilder::buildDerivationAsync( const StorePath & drvPath, const BasicDerivation & drv, const ServeProto::BuildOptions & options) { // Until we have C++23 std::move_only_function - auto conn = std::make_shared::Handle>(connections->get()); - (*conn)->putBuildDerivationRequest(*this, drvPath, drv, options); + auto conn = std::make_shared::Handle>(store->connections->get()); + (*conn)->putBuildDerivationRequest(*store, drvPath, drv, options); - return [this, conn]() -> BuildResult { return (*conn)->getBuildDerivationResponse(*this); }; + return [store = this->store, conn]() -> BuildResult { return (*conn)->getBuildDerivationResponse(*store); }; } -void LegacySSHStore::buildPaths( - const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) +std::variant +LegacySSHBuilder::buildPathsRaw(const std::vector & drvPaths, BuildMode buildMode) { - if (evalStore && evalStore.get() != this) - throw Error("building on an SSH store is incompatible with '--eval-store'"); - - auto conn(connections->get()); + auto conn(store->connections->get()); conn->to << ServeProto::Command::BuildPaths; Strings ss; @@ -229,11 +276,11 @@ void LegacySSHStore::buildPaths( auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit( overloaded{ - [&](const StorePathWithOutputs & s) { ss.push_back(s.to_string(*this)); }, + [&](const StorePathWithOutputs & s) { ss.push_back(s.to_string(*store)); }, [&](const StorePath & drvPath) { throw Error( "wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", - printStorePath(drvPath)); + store->printStorePath(drvPath)); }, [&](std::monostate) { throw Error( @@ -244,16 +291,108 @@ void LegacySSHStore::buildPaths( } conn->to << ss; - ServeProto::write(*this, *conn, buildSettings()); + ServeProto::write(*store, *conn, buildSettings()); conn->to.flush(); - auto status = CommonProto::Serialise::read(*this, {conn->from}); - if (auto * failure = std::get_if(&status)) { - std::string errorMsg; - conn->from >> errorMsg; - throw BuildError(*failure, std::move(errorMsg)); + auto status = CommonProto::Serialise::read(*store, {conn->from}); + return std::visit( + overloaded{ + [&](BuildResultSuccessStatus s) -> std::variant { return s; }, + [&](BuildResultFailureStatus s) -> std::variant { + std::string errorMsg; + conn->from >> errorMsg; + return BuildError{s, std::move(errorMsg)}; + }, + }, + status); +} + +void LegacySSHBuilder::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +{ + auto status = buildPathsRaw(drvPaths, buildMode); + if (auto * failure = std::get_if(&status)) + throw *failure; +} + +std::vector +LegacySSHBuilder::buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) +{ + auto status = buildPathsRaw(reqs, buildMode); + + std::vector results; + + // N.B. This logic is inspired by the fallback old protocol code in + // `RemoteBuilder::buildPathsWithResults`, which handles a very + // similar problem. + for (auto & req : reqs) { + std::visit( + overloaded{ + [&](const DerivedPath::Opaque & bo) { + results.push_back( + KeyedBuildResult{ + {.inner = std::visit( + overloaded{ + [](const BuildResultSuccessStatus & s) -> decltype(BuildResult::inner) { + return BuildResult::Success{.status = s}; + }, + [](const BuildError & e) -> decltype(BuildResult::inner) { return e; }, + }, + status)}, + /* .path = */ req, + }); + }, + [&](const DerivedPath::Built & bfd) { + if (auto * failure = std::get_if(&status)) { + results.push_back( + KeyedBuildResult{ + {.inner = *failure}, + /* .path = */ req, + }); + return; + } + + BuildResult::Success success{ + .status = std::get(status), + }; + + auto drvPath = resolveDerivedPath(*store, *bfd.drvPath); + auto built = resolveDerivedPath(*store, bfd); + for (auto & [output, outputPath] : built) { + auto outputId = DrvOutput{drvPath, output}; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + auto realisation = store->queryRealisation(outputId); + if (!realisation) + throw MissingRealisation(*store, outputId); + success.builtOutputs.emplace(output, *realisation); + } else { + success.builtOutputs.emplace( + output, + UnkeyedRealisation{ + .outPath = outputPath, + }); + } + } + + results.push_back( + KeyedBuildResult{ + {.inner = std::move(success)}, + /* .path = */ req, + }); + }, + }, + req.raw()); } + + return results; +} + +ref LegacySSHStore::getBuilder(std::shared_ptr evalStore) +{ + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + return make_ref( + ref(std::dynamic_pointer_cast(shared_from_this()))); } void LegacySSHStore::computeFSClosure( diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f66099005697..1b2e869f922f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,4 +1,5 @@ #include "nix/store/local-store.hh" +#include "nix/store/build.hh" #include "nix/store/globals.hh" #include "nix/store/path-references.hh" #include "nix/util/git.hh" @@ -1472,7 +1473,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) info->narHash.to_string(HashFormat::Nix32, true), current.hash.to_string(HashFormat::Nix32, true)); if (repair) - repairPath(i); + getBuilder()->repairPath(i); else errors = true; } else { @@ -1586,7 +1587,7 @@ void LocalStore::verifyPath( printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { - repairPath(path); + getBuilder()->repairPath(path); } catch (Error & e) { logWarning(e.info()); errors = true; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1ae9dae3573b..60ec3fb31e0f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -412,33 +412,6 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store req.raw()); } -OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) -{ - auto drvPath = resolveDerivedPath(store, *bfd.drvPath); - auto outputMap = deepQueryDerivationOutputMap(store, drvPath); - auto outputsLeft = std::visit( - overloaded{ - [&](const OutputsSpec::All &) { return StringSet{}; }, - [&](const OutputsSpec::Names & names) { return static_cast(names); }, - }, - bfd.outputs.raw); - for (auto iter = outputMap.begin(); iter != outputMap.end();) { - auto & outputName = iter->first; - if (bfd.outputs.contains(outputName)) { - outputsLeft.erase(outputName); - ++iter; - } else { - iter = outputMap.erase(iter); - } - } - if (!outputsLeft.empty()) - throw Error( - "derivation '%s' does not have an outputs %s", - store.printStorePath(drvPath), - concatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); - return outputMap; -} - } // namespace nix namespace nlohmann { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bd8aa9163ea2..b96a4326a32b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,3 +1,4 @@ +#include "nix/store/build.hh" #include "nix/store/path.hh" #include "nix/store/store-api.hh" #include "nix/util/file-content-address.hh" @@ -559,60 +560,95 @@ void RemoteStore::queryRealisationUncached( } } -void RemoteStore::copyDrvsFromEvalStore(const std::vector & paths, std::shared_ptr evalStore) +struct RemoteBuilder : Builder { - if (evalStore && evalStore.get() != this) { + ref store; + std::shared_ptr evalStore; + + RemoteBuilder(ref store, std::shared_ptr evalStore) + : store(store) + , evalStore(std::move(evalStore)) + { + } + +private: + + void copyDrvsFromEvalStore(const std::vector & paths); + +public: + + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override; + + std::vector + buildPathsWithResults(const std::vector & paths, BuildMode buildMode) override; + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + + void ensurePath(const StorePath & path) override; + + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override; +}; + +void RemoteBuilder::copyDrvsFromEvalStore(const std::vector & paths) +{ + if (evalStore && evalStore.get() != &*store) { /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ - RealisedPath::Set drvPaths2; + RealisedPath::Set drvPaths; for (const auto & i : paths) { std::visit( overloaded{ [&](const DerivedPath::Opaque & bp) { // Do nothing, path is hopefully there already }, - [&](const DerivedPath::Built & bp) { drvPaths2.insert(bp.drvPath->getBaseStorePath()); }, + [&](const DerivedPath::Built & bp) { drvPaths.insert(bp.drvPath->getBaseStorePath()); }, }, i.raw()); } - copyClosure(*evalStore, *this, drvPaths2); + copyClosure(*evalStore, *store, drvPaths); } } -void RemoteStore::buildPaths( - const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) +void RemoteBuilder::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { - copyDrvsFromEvalStore(drvPaths, evalStore); - - auto conn(getConnection()); + copyDrvsFromEvalStore(drvPaths); + auto conn(store->getConnection()); conn->to << WorkerProto::Op::BuildPaths; - WorkerProto::write(*this, *conn, drvPaths); + WorkerProto::write(*store, *conn, drvPaths); conn->to << buildMode; conn.processStderr(); readInt(conn->from); } -std::vector RemoteStore::buildPathsWithResults( - const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) +std::vector +RemoteBuilder::buildPathsWithResults(const std::vector & paths, BuildMode buildMode) { - copyDrvsFromEvalStore(paths, evalStore); + copyDrvsFromEvalStore(paths); - std::optional conn_(getConnection()); + std::optional conn_(store->getConnection()); auto & conn = *conn_; if (conn->protoVersion >= WorkerProto::Version{.number = {1, 34}}) { conn->to << WorkerProto::Op::BuildPathsWithResults; - WorkerProto::write(*this, *conn, paths); + WorkerProto::write(*store, *conn, paths); conn->to << buildMode; conn.processStderr(); - return WorkerProto::Serialise>::read(*this, *conn); + return WorkerProto::Serialise>::read(*store, *conn); } else { // Avoid deadlock. conn_.reset(); // Note: this throws an exception if a build/substitution // fails, but meh. - buildPaths(paths, buildMode, evalStore); + buildPaths(paths, buildMode); std::vector results; @@ -634,14 +670,14 @@ std::vector RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); - auto built = resolveDerivedPath(*this, bfd, &*evalStore); + auto drvPath = resolveDerivedPath(*store, *bfd.drvPath); + auto built = resolveDerivedPath(*store, bfd); for (auto & [output, outputPath] : built) { auto outputId = DrvOutput{drvPath, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto realisation = queryRealisation(outputId); + auto realisation = store->queryRealisation(outputId); if (!realisation) - throw MissingRealisation(*this, outputId); + throw MissingRealisation(*store, outputId); success.builtOutputs.emplace(output, *realisation); } else { success.builtOutputs.emplace( @@ -665,23 +701,34 @@ std::vector RemoteStore::buildPathsWithResults( } } -BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +BuildResult RemoteBuilder::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto conn(getConnection()); - conn->putBuildDerivationRequest(*this, &conn.daemonException, drvPath, drv, buildMode); + auto conn(store->getConnection()); + conn->putBuildDerivationRequest(*store, &conn.daemonException, drvPath, drv, buildMode); conn.processStderr(); - return WorkerProto::Serialise::read(*this, *conn); + return WorkerProto::Serialise::read(*store, *conn); } -void RemoteStore::ensurePath(const StorePath & path) +void RemoteBuilder::ensurePath(const StorePath & path) { - auto conn(getConnection()); + auto conn(store->getConnection()); conn->to << WorkerProto::Op::EnsurePath; - WorkerProto::write(*this, *conn, path); + WorkerProto::write(*store, *conn, path); conn.processStderr(); readInt(conn->from); } +void RemoteBuilder::repairPath(const StorePath & path) +{ + throw Unsupported("operation 'repairPath' is not supported by store '%s'", store->config.getHumanReadableURI()); +} + +ref RemoteStore::getBuilder(std::shared_ptr evalStore) +{ + return make_ref( + ref(std::dynamic_pointer_cast(shared_from_this())), std::move(evalStore)); +} + void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 9935f1f972ec..3fd9187e6219 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -1,9 +1,12 @@ #include "nix/store/restricted-store.hh" +#include "nix/store/build.hh" #include "nix/store/build-result.hh" #include "nix/store/submit-store.hh" +#include "nix/store/build/worker.hh" #include "nix/util/callback.hh" #include "nix/store/realisation.hh" #include "nix/store/local-store.hh" +#include "nix/util/error.hh" #include "nix/util/repair-flag.hh" namespace nix { @@ -110,8 +113,6 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void narFromPath(const StorePath & path, Sink & sink) override; - void ensurePath(const StorePath & path) override; - void registerDrvOutput(const Realisation & info) override; ref addToStoreScanning( @@ -124,20 +125,6 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void queryRealisationUncached( const DrvOutput & id, Callback> callback) noexcept override; - void - buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - - std::vector buildPathsWithResults( - const std::vector & paths, - BuildMode buildMode = bmNormal, - std::shared_ptr evalStore = nullptr) override; - - BuildResult - buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override - { - unsupported("buildDerivation"); - } - void addTempRoot(const StorePath & path) override {} void addIndirectRoot(const std::filesystem::path & path) override {} @@ -172,10 +159,42 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor { return NotTrusted; } + + ref getBuilder(std::shared_ptr evalStore) override + { + unreachable(); + } }; void RestrictedStore::anchor() {} +/** + * A builder that wraps an inner builder, adding restriction checks + * and dependency tracking for recursive Nix builds. + */ +struct RestrictedBuilder : Builder +{ + Worker & inner; + RestrictionContext & goal; + + RestrictedBuilder(Worker & inner, RestrictionContext & goal) + : inner(inner) + , goal(goal) + { + } + + void buildPaths(const std::vector & paths, BuildMode buildMode) override; + + std::vector + buildPathsWithResults(const std::vector & paths, BuildMode buildMode) override; + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + + void ensurePath(const StorePath & path) override; + + void repairPath(const StorePath & path) override; +}; + ref makeRestrictedStore(ref config, ref next, RestrictionContext & context) { return make_ref(config, next, context); @@ -251,10 +270,10 @@ void RestrictedStore::narFromPath(const StorePath & path, Sink & sink) Store::narFromPath(path, sink); } -void RestrictedStore::ensurePath(const StorePath & path) +void RestrictedBuilder::ensurePath(const StorePath & path) { if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", inner.store.printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ } @@ -290,36 +309,33 @@ void RestrictedStore::queryRealisationUncached( next->queryRealisation(id, std::move(callback)); } -void RestrictedStore::buildPaths( - const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) +void RestrictedBuilder::buildPaths(const std::vector & paths, BuildMode buildMode) { - for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) + for (auto & result : buildPathsWithResults(paths, buildMode)) result.tryThrowBuildError(); } -std::vector RestrictedStore::buildPathsWithResults( - const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) +std::vector +RestrictedBuilder::buildPathsWithResults(const std::vector & paths, BuildMode buildMode) { - assert(!evalStore); - if (buildMode != bmNormal) throw Error("unsupported build mode"); - StorePathSet newPaths; - std::set newRealisations; - for (auto & req : paths) { if (!goal.isAllowed(req)) - throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(inner.store)); } - auto results = next->buildPathsWithResults(paths, buildMode); + auto results = inner.buildPathsWithResults(paths, buildMode); + + StorePathSet newPaths; + std::set newRealisations; for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - if (auto * pathBuilt = std::get_if(&result.path)) { + if (auto * pathBuilt = std::get_if(&result.path)) { // TODO ugly extra IO - auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); + auto drvPath = resolveDerivedPath(inner.store, *pathBuilt->drvPath); for (auto & [outputName, output] : successP->builtOutputs) { newPaths.insert(output.outPath); newRealisations.insert( @@ -334,7 +350,7 @@ std::vector RestrictedStore::buildPathsWithResults( } StorePathSet closure; - next->computeFSClosure(newPaths, closure); + inner.store.computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); @@ -370,4 +386,20 @@ MissingPaths RestrictedStore::queryMissing(const std::vector & targ return res; } +ref makeRestrictedBuilder(Worker & inner, RestrictionContext & context) +{ + return make_ref(inner, context); +} + +BuildResult +RestrictedBuilder::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +{ + throw Unsupported("buildDerivation"); +} + +void RestrictedBuilder::repairPath(const StorePath & path) +{ + throw Unsupported("repairPath"); +} + } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fee7c8061efa..c6809a248fa6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,3 +1,4 @@ +#include "nix/store/build/worker.hh" #include "nix/util/logging.hh" #include "nix/util/signature/local-keys.hh" #include "nix/util/source-accessor.hh" @@ -6,6 +7,7 @@ #include "nix/store/realisation.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/store-open.hh" #include "nix/store/outputs-query.hh" #include "nix/util/util.hh" @@ -46,6 +48,8 @@ void InvalidStoreReference::anchor() {} void StoreConfigBase::anchor() {} +void Builder::anchor() {} + static std::string canonStoreDir(std::string path) { if (path.empty() || path[0] != '/') @@ -142,6 +146,13 @@ std::pair StoreDirConfig::toStorePath(std::string_view pat return {parseStorePath(path.substr(0, slash)), CanonPath{path.substr(slash)}}; } +ref Store::getBuilder(std::shared_ptr evalStore) +{ + auto store = ref(shared_from_this()); + auto evalStoreRef = evalStore ? ref(std::move(evalStore)) : store; + return make_ref(store, evalStoreRef); +} + std::filesystem::path Store::followLinksToStore(std::string_view _path) const { auto path = absPath(std::string(_path)); @@ -734,7 +745,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector subs; for (auto & p : missing.willSubstitute) subs.emplace_back(DerivedPath::Opaque{p}); - buildPaths(subs); + getBuilder()->buildPaths(subs, bmNormal); } catch (Error & e) { logWarning(e.info()); } @@ -1176,7 +1187,7 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optionalensurePath(drvPath); return readDerivation(drvPath); } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 3b02dfc4f172..24c2437fd57f 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -8,7 +8,6 @@ #include "nix/util/util.hh" #include "nix/util/archive.hh" #include "nix/util/git.hh" -#include "nix/store/daemon.hh" #include "nix/util/topo-sort.hh" #include "nix/store/build/child.hh" #include "nix/util/unix-domain-socket.hh" @@ -829,10 +828,9 @@ void DerivationBuilderImpl::startDaemon() auto doneFlag = make_ref(); - auto workerThread = std::thread([doneFlag, store, remote{std::move(remote)}]() { + auto workerThread = std::thread([this, doneFlag, store, remote{std::move(remote)}]() { try { - daemon::processConnection( - store, FdSource(remote.get()), FdSink(remote.get()), NotTrusted, daemon::Recursive); + miscMethods->processDaemonConnection(store, FdSource(remote.get()), FdSink(remote.get()), *this); debug("terminated daemon connection"); } catch (const Interrupted &) { debug("interrupted daemon connection"); diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 8f8f21a8c39c..648f05a489c1 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -21,6 +21,7 @@ #include "nix/util/strings.hh" #include "nix/store/derivations.hh" #include "nix/store/local-store.hh" +#include "nix/store/build.hh" #include "nix/cmd/legacy.hh" #include "nix/util/experimental-features.hh" #include "nix/store/globals.hh" @@ -338,7 +339,7 @@ static int main_build_remote(int argc, char ** argv) // output ids, which break CA derivations if (!drv.inputDrvs.map.empty()) drv.inputSrcs = store->parseStorePathSet(inputs); - optResult = sshStore->buildDerivation(*drvPath, static_cast(drv)); + optResult = sshStore->getBuilder()->buildDerivation(*drvPath, static_cast(drv)); auto & result = *optResult; if (auto * failureP = result.tryGetFailure()) { if (settings.keepFailed) { @@ -353,7 +354,7 @@ static int main_build_remote(int argc, char ** argv) } } else { copyClosure(*store, *sshStore, StorePathSet{*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({DerivedPath::Built{ + auto res = sshStore->getBuilder()->buildPathsWithResults({DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All{}, }}); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 4808a2fc716f..80dfeebb99a1 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -2,6 +2,7 @@ #include "nix/cmd/command-installable-value.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/local-fs-store.hh" #include "nix/expr/eval-inline.hh" #include "nix/store/globals.hh" @@ -111,7 +112,7 @@ struct CmdBundle : InstallableValueCommand auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); - store->buildPaths({ + store->getBuilder()->buildPaths({ DerivedPath::Built{ .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}, diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1db0363be79c..350b4d5bbd56 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -5,6 +5,7 @@ #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/globals.hh" #include "nix/store/outputs-spec.hh" #include "nix/store/outputs-query.hh" @@ -290,13 +291,12 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto shellDrvPath = evalStore->writeDerivation(drv); /* Build the derivation. */ - store->buildPaths( + store->getBuilder(evalStore)->buildPaths( {DerivedPath::Built{ .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All{}, }}, - bmNormal, - evalStore); + bmNormal); // `get-env.sh` will write its JSON output to an arbitrary output // path, so return the first non-empty output path. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a665e717e1f9..c0dd33822058 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -22,6 +22,7 @@ #include "nix/util/users.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/store/local-fs-store.hh" +#include "nix/store/build.hh" #include "nix/store/globals.hh" #include @@ -823,9 +824,8 @@ struct CmdFlakeCheck : FlakeCommand, MixPrintOutPaths, MixOutLinkBase } Activity act(*logger, lvlInfo, actUnknown, fmt("running %d flake checks", toBuild.size())); - // once we get rid of the temporary hack above, this tenary operator will also go away - results = store->buildPathsWithResults((printOutputPaths || outLink) ? drvPaths : toBuild); + results = store->getBuilder()->buildPathsWithResults((printOutputPaths || outLink) ? drvPaths : toBuild); // Report build failures with attribute paths for (auto & result : results) { diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index c485a084746c..b7c9531b1481 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -28,6 +28,7 @@ #include "nix/util/users.hh" #include "nix/cmd/network-proxy.hh" #include "nix/cmd/compatibility-settings.hh" +#include "nix/store/build.hh" #include "nix/util/fun.hh" #include "man-pages.hh" @@ -449,7 +450,7 @@ static void main_nix_build(int argc, char ** argv) printMissing(ref(store), paths); if (!dryRun) - store->buildPaths(paths, buildMode, evalStore); + store->getBuilder(evalStore)->buildPaths(paths, buildMode); }; if (isNixShell) { diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 8568b1c8d59c..671fa46f45f5 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -12,6 +12,7 @@ #include "nix/store/path-with-outputs.hh" #include "nix/main/shared.hh" #include "nix/store/store-open.hh" +#include "nix/store/build.hh" #include "nix/store/local-fs-store.hh" #include "user-env.hh" #include "nix/expr/value-to-json.hh" @@ -793,7 +794,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) printMissing(globals.state->store, paths); if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); + globals.state->store->getBuilder()->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); debug("switching to new user environment"); auto generation = createGeneration(*store2, globals.profile, drv.queryOutPath()); diff --git a/src/nix/nix-env/user-env.cc b/src/nix/nix-env/user-env.cc index 9d0cdee48fe5..5dbcd93dfa38 100644 --- a/src/nix/nix-env/user-env.cc +++ b/src/nix/nix-env/user-env.cc @@ -1,6 +1,7 @@ #include "user-env.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/local-fs-store.hh" #include "nix/main/shared.hh" @@ -44,7 +45,7 @@ bool createUserEnv( drvsToBuild.push_back({*drvPath}); debug("building user environment dependencies"); - state.store->buildPaths(toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); + state.store->getBuilder()->buildPaths(toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ StorePathSet references; @@ -79,7 +80,7 @@ bool createUserEnv( /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ state.store->addTempRoot(*j.second); - state.store->ensurePath(*j.second); + state.store->getBuilder()->ensurePath(*j.second); references.insert(*j.second); } @@ -154,7 +155,7 @@ bool createUserEnv( debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); - state.store->buildPaths(toDerivedPaths(topLevelDrvs), state.repair ? bmRepair : bmNormal); + state.store->getBuilder()->buildPaths(toDerivedPaths(topLevelDrvs), state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast(); diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 77c251944f38..3ccec3241597 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -21,6 +21,7 @@ #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/error.hh" #include "nix/store/gc-store.hh" +#include "nix/store/build.hh" #include "man-pages.hh" @@ -76,7 +77,7 @@ static std::set realisePath(StorePathWithOutputs path, bo if (path.path.isDerivation()) { if (build) - store->buildPaths({path.toDerivedPath()}); + store->getBuilder()->buildPaths({path.toDerivedPath()}); auto outputPaths = deepQueryDerivationOutputMap(*store, path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -113,7 +114,7 @@ static std::set realisePath(StorePathWithOutputs path, bo else { if (build) - store->ensurePath(path.path); + store->getBuilder()->ensurePath(path.path); else if (!store->isValidPath(path.path)) throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path)); if (store2) { @@ -173,7 +174,7 @@ static void opRealise(Strings opFlags, Strings opArgs) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(toDerivedPaths(paths), buildMode); + store->getBuilder()->buildPaths(toDerivedPaths(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -862,7 +863,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - store->repairPath(store->followLinksToStorePath(i)); + store->getBuilder()->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking @@ -1009,7 +1010,7 @@ static void opServe(Strings opFlags, Strings opArgs) #ifndef _WIN32 // TODO figure out if Windows needs something similar MonitorFdHup monitor(in.fd); #endif - store->buildPaths(toDerivedPaths(paths)); + store->getBuilder()->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { assert(e.info().status); @@ -1032,7 +1033,7 @@ static void opServe(Strings opFlags, Strings opArgs) #ifndef _WIN32 // TODO figure out if Windows needs something similar MonitorFdHup monitor(in.fd); #endif - auto status = store->buildDerivation(drvPath, drv); + auto status = store->getBuilder()->buildDerivation(drvPath, drv); ServeProto::write(*store, wconn, status); break; diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc index a6eee6dfbf2c..6c91d1959ac1 100644 --- a/src/nix/store-repair.cc +++ b/src/nix/store-repair.cc @@ -1,5 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" namespace nix { @@ -20,7 +21,7 @@ struct CmdStoreRepair : StorePathsCommand void run(ref store, StorePaths && storePaths) override { for (auto & path : storePaths) - store->repairPath(path); + store->getBuilder()->repairPath(path); } }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 1b2873d7f9c0..35b9771ef09a 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -3,6 +3,7 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/filetransfer.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" @@ -113,7 +114,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath))); - store->ensurePath(storePath); + store->getBuilder()->ensurePath(storePath); } { diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index cab02d799d79..9ba21306ab5e 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -1,5 +1,6 @@ #include "nix/store/globals.hh" #include "nix/store/store-open.hh" +#include "nix/store/build.hh" #include "nix/store/build-result.hh" #include @@ -24,7 +25,7 @@ int main(int argc, char ** argv) std::vector paths{DerivedPath::Built{ .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)), .outputs = OutputsSpec::Names{"out"}}}; - const auto results = store->buildPathsWithResults(paths, bmNormal, store); + const auto results = store->getBuilder()->buildPathsWithResults(paths, bmNormal); for (const auto & result : results) { if (auto * successP = result.tryGetSuccess()) { From e959fe544bc80f2c1d39a4d6ef36dd41c1522195 Mon Sep 17 00:00:00 2001 From: aetosdios27 Date: Wed, 15 Jul 2026 16:59:52 +0530 Subject: [PATCH 353/364] Appropriate exclusion of rl-next.md when officialRelease = true Assisted-by: Codex (GPT-5) --- doc/manual/source/meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/meson.build b/doc/manual/source/meson.build index 294d57ad9f9c..115783c6b24a 100644 --- a/doc/manual/source/meson.build +++ b/doc/manual/source/meson.build @@ -8,10 +8,10 @@ summary_rl_next = custom_target( 'pipefail', '-c', ''' - if [ -e "@INPUT@" ]; then + if [ '@0@' = 'false' ]; then echo ' - [Upcoming release](release-notes/rl-next.md)' fi - ''', + '''.format(official_release), ], input : [ rl_next_generated, From e892619b1436d1e6426b2edcdaced0797408caa1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Nov 2025 00:43:39 -0500 Subject: [PATCH 354/364] libstore: parameterize the inputs of `Derivation` Working with templates adds some overhead, but parameterizing the inputs is an important prerequisite for storing the options inside the derivation. Subsequent commits can clean this up significantly. Co-authored-by: Amaan Qureshi Assisted-by: Claude:opus-4.8 --- src/libexpr/primops.cc | 8 +- .../derivation-advanced-attrs.cc | 12 +- .../derivation/external-formats.cc | 8 +- src/libstore-tests/derivation/invariants.cc | 4 +- src/libstore-tests/derivations.cc | 12 +- src/libstore-tests/outputs-query.cc | 4 +- src/libstore-tests/worker-substitution.cc | 4 +- .../build/derivation-building-goal.cc | 82 +--- src/libstore/build/derivation-env-desugar.cc | 2 +- src/libstore/build/derivation-goal.cc | 53 ++- .../build/derivation-resolution-goal.cc | 4 +- src/libstore/build/entry-points.cc | 2 +- src/libstore/build/worker.cc | 7 +- src/libstore/daemon.cc | 4 +- src/libstore/derivation-options.cc | 14 +- src/libstore/derivations.cc | 390 ++++++++++-------- src/libstore/globals.cc | 2 +- .../store/build/derivation-building-goal.hh | 21 +- .../store/build/derivation-building-misc.hh | 9 +- .../nix/store/build/derivation-env-desugar.hh | 10 +- .../include/nix/store/build/worker.hh | 4 +- .../include/nix/store/derivation-options.hh | 15 +- src/libstore/include/nix/store/derivations.hh | 149 ++++--- .../nix/store/downstream-placeholder.hh | 2 +- .../include/nix/store/local-settings.hh | 2 +- .../include/nix/store/outputs-query.hh | 2 +- src/libstore/include/nix/store/store-api.hh | 9 +- src/libstore/misc.cc | 6 +- src/libstore/outputs-query.cc | 6 +- src/libstore/store-api.cc | 2 +- src/libutil/include/nix/util/json-impls.hh | 11 +- src/nix/build-remote/build-remote.cc | 32 +- src/nix/develop.cc | 2 +- src/nix/nix-build/nix-build.cc | 8 +- 34 files changed, 504 insertions(+), 398 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cbe994e78193..2428d98768a1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1749,18 +1749,18 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName StorePathSet refs; state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { - drv.inputSrcs.insert(j); + drv.inputs.srcs.insert(j); if (j.isDerivation()) { - drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames(); + drv.inputs.drvs.map[j].value = state.store->readDerivation(j).outputNames(); } } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); + drv.inputs.drvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { state.ensureLazyPathCopied(o.path); - drv.inputSrcs.insert(o.path); + drv.inputs.srcs.insert(o.path); }, }, c.raw); diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 6d2c053c0d65..5a20c4f91539 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -37,7 +37,7 @@ class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, this->readTest(fileName, [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures); }); } @@ -53,7 +53,7 @@ class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, this->readTest(fileName, [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_EQ(options, expected); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures); @@ -185,7 +185,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(!got.structuredAttrs); @@ -229,7 +229,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(!got.structuredAttrs); @@ -325,7 +325,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(got.structuredAttrs); @@ -374,7 +374,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(got.structuredAttrs); diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index e31b3e85442f..d1ccc203c97a 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -188,10 +188,10 @@ MAKE_TEST_P(DerivationJsonAtermTest); INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { Derivation drv; drv.name = "simple-derivation"; - drv.inputSrcs = { + drv.inputs.srcs = { StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; - drv.inputDrvs = { + drv.inputs.drvs = { .map = { { @@ -232,10 +232,10 @@ Derivation makeDynDepDerivation() { Derivation drv; drv.name = "dyn-dep-derivation"; - drv.inputSrcs = { + drv.inputs.srcs = { StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, }; - drv.inputDrvs = { + drv.inputs.drvs = { .map = { { diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index e825655d66f4..edffe176d040 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -216,7 +216,7 @@ TEST_F(FillInOutputPathsTest, preservesDeferredWithInputDrvs) {"out", ""}, }; // Add the real input derivation dependency - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Serialize before state checkpointJson("depends-on-drv-pre", drv); @@ -252,7 +252,7 @@ TEST_F(FillInOutputPathsTest, throwsOnPatWhenShouldBeDeffered) {"out", ""}, }; // Add the real input derivation dependency - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Serialize before state checkpointJson("bad-depends-on-drv-pre", drv); diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc index 60b86f571205..39449baf9128 100644 --- a/src/libstore-tests/derivations.cc +++ b/src/libstore-tests/derivations.cc @@ -174,7 +174,7 @@ TEST_F(TryResolveTest, withInputs) drv.platform = "x86_64-linux"; drv.builder = "/bin/bash"; drv.outputs = multiOutputs; - drv.inputDrvs = { + drv.inputs.drvs = { .map = { {dep1DrvPath, {.value = {"out", "dev"}}}, {dep2DrvPath, {.value = {"out"}}}, @@ -219,7 +219,7 @@ TEST_F(TryResolveTest, withInputs) expected.platform = "x86_64-linux"; expected.builder = "/bin/bash"; expected.outputs = multiOutputs; - expected.inputSrcs = {dep1OutPath, dep1DevPath, dep2OutPath}; + expected.inputs = {dep1OutPath, dep1DevPath, dep2OutPath}; expected.env = { {"DEP1_OUT", "prefix-" + store->printStorePath(dep1OutPath) + "-suffix"}, {"DEP1_DEV", store->printStorePath(dep1DevPath)}, @@ -243,7 +243,7 @@ TEST_F(TryResolveTest, resolutionFailure) drv.platform = "x86_64-linux"; drv.builder = "/bin/bash"; drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; BuildTrace buildTrace; @@ -273,7 +273,7 @@ void TryResolveTest::exportRefGraphSubpathTest( nix::checkpointJson(*this, std::string{stem} + "-before", drv); - auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputDrvs, drv.env, parsed, true); + auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputs.drvs, drv.env, parsed, true); nix::checkpointJson(*this, std::string{stem} + "-before-options", options); @@ -333,7 +333,7 @@ TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) drv.platform = "x86_64-linux"; drv.builder = "/bin/bash"; drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; drv.env = { {"exportReferencesGraph", "refs " + placeholder + "/foo"}, }; @@ -351,7 +351,7 @@ TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) drv.platform = "x86_64-linux"; drv.builder = "/bin/bash"; drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; drv.structuredAttrs = StructuredAttrs{{ {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, }}; diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc index 4be25e2ad3b8..de585af61987 100644 --- a/src/libstore-tests/outputs-query.cc +++ b/src/libstore-tests/outputs-query.cc @@ -72,8 +72,8 @@ TEST_F(OutputsQueryTest, fibonacciChainQueryCount) // d_i depends on d_{i-1} and d_{i-2} for (size_t i = 2; i <= N; ++i) { Derivation drv = makeLeafDrv("d" + std::to_string(i)); - drv.inputDrvs.map[drvPaths[i - 1]].value.insert("out"); - drv.inputDrvs.map[drvPaths[i - 2]].value.insert("out"); + drv.inputs.drvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputs.drvs.map[drvPaths[i - 2]].value.insert("out"); drvPaths.push_back(store->writeDerivation(drv)); } diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 3534d44d0d8f..2a19cf7580a3 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -329,7 +329,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) }, }; // Add the dependency derivation as an input - rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + rootDrv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Write the root derivation to the destination store auto rootDrvPath = dummyStore->writeDerivation(rootDrv); @@ -345,7 +345,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) ASSERT_TRUE(resolvedRootDrv); // Write the resolved derivation to the substituter - auto resolvedRootDrvPath = substituter->writeDerivation(Derivation{*resolvedRootDrv}); + auto resolvedRootDrvPath = substituter->writeDerivation(resolvedRootDrv->unresolve()); // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 386f591d842b..04f1c15b1714 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -32,8 +32,8 @@ namespace nix { DerivationBuildingGoal::DerivationBuildingGoal( - const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode, bool storeDerivation) - : Goal(worker, gaveUpOnSubstitution(storeDerivation)) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) + : Goal(worker, gaveUpOnSubstitution()) , drvPath(drvPath) , drv{std::move(drv)} , buildMode(buildMode) @@ -53,7 +53,8 @@ std::string DerivationBuildingGoal::key() return "dd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv) +template +std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv) { std::string msg; StorePathSet expectedOutputPaths; @@ -68,6 +69,9 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } +template std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv); +template std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv); + namespace { struct LogSink : Sink @@ -146,7 +150,7 @@ static std::unique_ptr runPostBuildHook( /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) +Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() { Goals waitees; @@ -157,13 +161,13 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) are (resolved) derivation outputs in a resolved derivation. */ if (&worker.evalStore != &worker.store) { RealisedPath::Set inputSrcs; - for (auto & i : drv->inputSrcs) + for (auto & i : drv->inputs) if (worker.evalStore.isValidPath(i)) inputSrcs.insert(i); copyClosure(worker.evalStore, worker.store, inputSrcs); } - for (auto & i : drv->inputSrcs) { + for (auto & i : drv->inputs) { if (worker.store.isValidPath(i)) continue; if (!worker.settings.useSubstitutes) @@ -194,47 +198,8 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) /* Determine the full set of input paths. */ - if (storeDerivation) { - assert(drv->inputDrvs.map.empty()); - /* Store the resolved derivation, as part of the record of - what we're actually building */ - worker.store.writeDerivation(*drv); - } - StorePathSet inputPaths; - - { - /* If we get this far, we know no dynamic drvs inputs */ - - for (auto & [depDrvPath, depNode] : drv->inputDrvs.map) { - for (auto & outputName : depNode.value) { - /* Don't need to worry about `inputGoals`, because - impure derivations are always resolved above. Can - just use DB. This case only happens in the (older) - input addressed and fixed output derivation cases. */ - auto outMap = [&] { - for (auto * drvStore : {&worker.evalStore, &worker.store}) - if (drvStore->isValidPath(depDrvPath)) - return deepQueryDerivationOutputMap(worker.store, depDrvPath, drvStore); - assert(false); - }(); - - auto outMapPath = outMap.find(outputName); - if (outMapPath == outMap.end()) { - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), - outputName, - worker.store.printStorePath(depDrvPath)); - } - - worker.store.computeFSClosure(outMapPath->second, inputPaths); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + worker.store.computeFSClosure(drv->inputs, inputPaths); debug("added input paths %s", concatMapStringsSep(", ", inputPaths, [&](auto & p) { return "'" + worker.store.printStorePath(p) + "'"; @@ -339,35 +304,12 @@ static BuildError reject(const LocalBuildRejection & rejection, std::string_view Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) { auto drvOptions = [&] { - DerivationOptions temp; try { - temp = - derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); + return derivationOptionsFromStructuredAttrs(worker.store, drv->env, get(drv->structuredAttrs)); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); throw; } - - auto res = tryResolve( - temp, - [&](ref drvPath, const std::string & outputName) -> std::optional { - try { - return resolveDerivedPath( - worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore); - } catch (Error &) { - return std::nullopt; - } - }); - - /* The derivation must have all of its inputs gotten this point, - so the resolution will surely succeed. - - (Actually, we shouldn't even enter this goal until we have a - resolved derivation, or derivation with only input addressed - transitive inputs, so this should be a no-opt anyways.) - */ - assert(res); - return *res; }(); std::map initialOutputs; diff --git a/src/libstore/build/derivation-env-desugar.cc b/src/libstore/build/derivation-env-desugar.cc index 75b62c116502..ff19472d0122 100644 --- a/src/libstore/build/derivation-env-desugar.cc +++ b/src/libstore/build/derivation-env-desugar.cc @@ -19,7 +19,7 @@ std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fil DesugaredEnv DesugaredEnv::create( Store & store, - const Derivation & drv, + const BasicDerivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths) { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 24f111c8b5d1..19281191cc7f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,7 +52,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto drvOptions = [&]() -> DerivationOptions { try { return derivationOptionsFromStructuredAttrs( - worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); + worker.store, drv->inputs.drvs, drv->env, get(drv->structuredAttrs)); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); throw; @@ -164,7 +164,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, - make_ref(drvResolved), + make_ref(drvResolved.unresolve()), wantedOutput, buildMode, /*storeDerivation=*/true); @@ -230,7 +230,54 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* Give up on substitution for the output we want, actually build this derivation */ - auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); + /* Project down to the `BasicDerivation` the builder consumes, + adding the outputs of the input derivations to the input + sources. */ + auto resolvedDrv = make_ref(drv->mapInputs([&](const FullInputs & inputs) { + auto srcs = inputs.srcs; + for (auto & [depDrvPath, depNode] : inputs.drvs.map) { + for (auto & outputName : depNode.value) { + /* Don't need to worry about `inputGoals`, because + impure derivations are always resolved above. Can + just use DB. This case only happens in the (older) + input addressed and fixed output derivation cases. */ + auto outMap = [&] { + for (auto * drvStore : {&worker.evalStore, &worker.store}) + if (drvStore->isValidPath(depDrvPath)) + return deepQueryDerivationOutputMap(worker.store, depDrvPath, drvStore); + assert(false); + }(); + auto outMapPath = outMap.find(outputName); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), + outputName, + worker.store.printStorePath(depDrvPath)); + } + srcs.insert(outMapPath->second); + } + } + return srcs; + })); + + if (storeDerivation) { + assert(drv->inputs.drvs.map.empty()); + /* `writeDerivation` checks the derivation's references are valid, + so the eval store's sources must be copied over first. */ + if (&worker.evalStore != &worker.store) { + RealisedPath::Set inputSrcs; + for (auto & i : resolvedDrv->inputs) + if (worker.evalStore.isValidPath(i)) + inputSrcs.insert(i); + copyClosure(worker.evalStore, worker.store, inputSrcs); + } + /* Store the resolved derivation, as part of the record of + what we're actually building */ + worker.store.writeDerivation(resolvedDrv->unresolve()); + } + + auto g = worker.makeDerivationBuildingGoal(drvPath, resolvedDrv, buildMode); /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 6f55ea29d96e..eac55a2e8f3f 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -51,7 +51,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); }; - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { + for (const auto & [inputDrvPath, inputNode] : drv->inputs.drvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() @@ -133,7 +133,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() } assert(attempt); - auto pathResolved = computeStorePath(worker.store, Derivation{*attempt}); + auto pathResolved = computeStorePath(worker.store, attempt->unresolve()); auto msg = fmt("resolved derivation: '%s' -> '%s'", diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e4fceb364a85..fdbe5f269821 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -107,7 +107,7 @@ std::vector Worker::buildPathsWithResults(const std::vector Worker::makeDerivationBuildingGoal( - const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation) +std::shared_ptr +Worker::makeDerivationBuildingGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { - return initGoalIfNeeded( - derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode, storeDerivation); + return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode); } std::shared_ptr diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 23df48bfc29d..0c5673ea78d8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -650,9 +650,7 @@ static void performOp( paths. */ assert(drvType.isCA()); - Derivation drv2; - static_cast(drv2) = drv; - drvPath = store->writeDerivation(Derivation{drv2}); + drvPath = store->writeDerivation(drv.unresolve()); } auto res = builder.buildDerivation(drvPath, drv, buildMode); diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index ab73e921fcea..36cf9116c833 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -358,7 +358,8 @@ DerivationOptions derivationOptionsFromStructuredAttrs( } template -StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const +template +StringSet DerivationOptions::getRequiredSystemFeatures(const DerivationT & drv) const { // FIXME: cache this? StringSet res; @@ -376,11 +377,20 @@ bool DerivationOptions::substitutesAllowed(const WorkerSettings & workerS } template -bool DerivationOptions::useUidRange(const BasicDerivation & drv) const +template +bool DerivationOptions::useUidRange(const DerivationT & drv) const { return getRequiredSystemFeatures(drv).count("uid-range"); } +// Explicit instantiations for member function templates +template StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation &) const; +template StringSet DerivationOptions::getRequiredSystemFeatures(const Derivation &) const; +template StringSet DerivationOptions::getRequiredSystemFeatures(const Derivation &) const; + +template bool DerivationOptions::useUidRange(const BasicDerivation &) const; +template bool DerivationOptions::useUidRange(const Derivation &) const; + std::optional> tryResolve( const DerivationOptions & drvOptions, fun(ref drvPath, const std::string & outputName)> diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d25dc19cd493..17f2285b33e6 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,10 +15,6 @@ namespace nix { -BasicDerivation::~BasicDerivation() {} - -Derivation::~Derivation() {} - std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { @@ -101,15 +97,26 @@ bool DerivationType::isImpure() const raw); } -bool BasicDerivation::isBuiltin() const +bool isBuiltin(const BasicDerivation & drv) +{ + return drv.builder.substr(0, 8) == "builtin:"; +} + +template +bool DerivationT::isBuiltin() const { return builder.substr(0, 8) == "builtin:"; } +// Forward declaration of specialization +template<> +std::string DerivationT::unparse( + const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const; + static auto infoForDerivation(const StoreDirConfig & store, const Derivation & drv) { - auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs.map) + auto references = drv.inputs.srcs; + for (auto & i : drv.inputs.drvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -489,13 +496,13 @@ Derivation parseDerivation( expect(str, '('); auto drvPath = parsePath(str); expect(str, ','); - drv.inputDrvs.map.insert_or_assign( + drv.inputs.drvs.map.insert_or_assign( store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ')'); } expect(str, ','); - drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + drv.inputs.srcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ','); drv.platform = parseString(str).toOwned(); expect(str, ','); @@ -640,13 +647,14 @@ static void unparseDerivedPathMapNode( static bool hasDynamicDrvDep(const Derivation & drv) { return std::find_if( - drv.inputDrvs.map.begin(), - drv.inputDrvs.map.end(), + drv.inputs.drvs.map.begin(), + drv.inputs.drvs.map.end(), [](auto & kv) { return !kv.second.childMap.empty(); }) - != drv.inputDrvs.map.end(); + != drv.inputs.drvs.map.end(); } -std::string Derivation::unparse( +template<> +std::string DerivationT::unparse( const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { using namespace std::literals::string_view_literals; @@ -736,7 +744,7 @@ std::string Derivation::unparse( s += ')'; } } else { - for (auto & [drvPath, childMap] : inputDrvs.map) { + for (auto & [drvPath, childMap] : inputs.drvs.map) { if (first) first = false; else @@ -749,7 +757,7 @@ std::string Derivation::unparse( } s += "],"sv; - auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow + auto paths = store.printStorePathSet(inputs.srcs); // FIXME: slow printUnquotedStrings(s, paths.begin(), paths.end()); s += ','; @@ -808,7 +816,8 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName) return res; } -DerivationType BasicDerivation::type() const +template +DerivationType DerivationT::type() const { using namespace std::literals::string_view_literals; @@ -945,7 +954,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* For other derivations, replace the inputs paths with recursive calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; - for (auto & [drvPath, node] : drv.inputDrvs.map) { + for (auto & [drvPath, node] : drv.inputs.drvs.map) { /* Need to build and resolve dynamic derivations first */ if (!node.childMap.empty()) { return DrvHashModulo::DeferredDrv{}; @@ -993,7 +1002,8 @@ static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } -StringSet BasicDerivation::outputNames() const +template +StringSet DerivationT::outputNames() const { StringSet names; for (auto & i : outputs) @@ -1001,7 +1011,8 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const +template +DerivationOutputsAndOptPaths DerivationT::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; for (auto & [outputName, output] : outputs) @@ -1010,7 +1021,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirC return outsAndOptPaths; } -std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) +template +std::string_view DerivationT::nameFromPath(const StorePath & drvPath) { drvPath.requireDerivation(); auto nameWithSuffix = drvPath.name(); @@ -1030,7 +1042,7 @@ Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivati drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = in}); + drv.inputs = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = in}); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -1074,7 +1086,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, i.second.raw); } - CommonProto::write(store, CommonProto::WriteConn{.to = out}, drv.inputSrcs); + CommonProto::write(store, CommonProto::WriteConn{.to = out}, drv.inputs); out << drv.platform << drv.builder << drv.args; auto writeEnv = [&](const StringPairs atermEnv) { @@ -1101,7 +1113,8 @@ std::string hashPlaceholder(const OutputNameView outputName) .to_string(HashFormat::Nix32, false); } -void BasicDerivation::applyRewrites(const StringMap & rewrites) +template +void DerivationT::applyRewrites(const StringMap & rewrites) { if (rewrites.empty()) return; @@ -1131,10 +1144,17 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } +template<> +Derivation DerivationT::unresolve() const +{ + return mapInputs([](const StorePathSet & inputs) -> FullInputs { return {.srcs = inputs, .drvs = {}}; }); +} + +template<> bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ - if (inputDrvs.map.empty()) + if (inputs.drvs.map.empty()) return false; auto drvType = type(); @@ -1159,22 +1179,13 @@ bool Derivation::shouldResolve() const /* Also need to resolve if any inputs are outputs of dynamic derivations. */ bool hasDynamicInputs = std::ranges::any_of( - inputDrvs.map.begin(), inputDrvs.map.end(), [](auto & pair) { return !pair.second.childMap.empty(); }); + inputs.drvs.map.begin(), inputs.drvs.map.end(), [](auto & pair) { return !pair.second.childMap.empty(); }); return typeNeedsResolve || hasDynamicInputs; } -std::optional Derivation::tryResolve(Store & store, Store * evalStore) const -{ - return tryResolve( - store, [&](ref drvPath, const std::string & outputName) -> std::optional { - try { - return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore); - } catch (Error &) { - return std::nullopt; - } - }); -} +template +static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName); static bool tryResolveInput( const StoreDirConfig & store, @@ -1221,20 +1232,49 @@ static bool tryResolveInput( return true; } -std::optional Derivation::tryResolve( +// Forward declaration of specialization +template<> +std::optional DerivationT::tryResolve( + Store & store, + fun(ref drvPath, const std::string & outputName)> + queryResolutionChain) const; + +template<> +std::optional DerivationT::tryResolve(Store & store, Store * evalStore) const +{ + return tryResolve( + store, [&](ref drvPath, const std::string & outputName) -> std::optional { + try { + return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore); + } catch (Error &) { + return std::nullopt; + } + }); +} + +template<> +std::optional DerivationT::tryResolve( Store & store, fun(ref drvPath, const std::string & outputName)> queryResolutionChain) const { - BasicDerivation resolved{*this}; + BasicDerivation resolved{ + .outputs = outputs, + .inputs = inputs.srcs, + .platform = platform, + .builder = builder, + .args = args, + .env = env, + .structuredAttrs = structuredAttrs, + .name = name, + }; - // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputNode] : inputDrvs.map) + for (auto & [inputDrv, inputNode] : inputs.drvs.map) if (!tryResolveInput( store, - resolved.inputSrcs, + resolved.inputs, inputRewrites, nullptr, make_ref(SingleDerivedPath::Opaque{inputDrv}), @@ -1244,11 +1284,9 @@ std::optional Derivation::tryResolve( resolved.applyRewrites(inputRewrites); - Derivation resolved2{std::move(resolved)}; - - resolved2.fillInOutputPaths(store); + processDerivationOutputPaths(store, resolved, resolved.name); - return resolved2; + return resolved; } /** @@ -1280,7 +1318,11 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string auto hashModulo = [&]() -> const auto & { if (!hashModulo_) { // somewhat expensive so we do lazily - hashModulo_ = hashDerivationModulo(store, drv, true); + if constexpr (std::is_same_v, Derivation>) { + hashModulo_ = hashDerivationModulo(store, drv, true); + } else { + hashModulo_ = hashDerivationModulo(store, drv.unresolve(), true); + } } return *hashModulo_; }; @@ -1397,7 +1439,8 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string drv.type(); } -void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +template +void DerivationT::checkInvariants(Store & store, const StorePath & drvPath) const { assert(drvPath.isDerivation()); std::string drvName(drvPath.name()); @@ -1415,16 +1458,25 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const } } +template<> +void BasicDerivation::checkInvariants(Store & store) const +{ + processDerivationOutputPaths(store, *this, name); +} + +template<> void Derivation::checkInvariants(Store & store) const { processDerivationOutputPaths(store, *this, name); } +template<> void Derivation::fillInOutputPaths(Store & store) { processDerivationOutputPaths(store, *this, name); } +template<> Derivation Derivation::parseJsonAndValidate(Store & store, const nlohmann::json & json) { auto drv = static_cast(json); @@ -1443,6 +1495,10 @@ Derivation Derivation::parseJsonAndValidate(Store & store, const nlohmann::json const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); +// Explicit template instantiations +template struct DerivationT; +template struct DerivationT; + } // namespace nix namespace nlohmann { @@ -1544,14 +1600,40 @@ nix::DerivationOutput adl_serializer::from_json( } } -static void inputSrcsToJson(json & res, const nix::StorePathSet & inputSrcs) +static void inputsToJson(json & res, const nix::StorePathSet & inputs) { res = nlohmann::json::array(); - for (auto & input : inputSrcs) + for (auto & input : inputs) res.emplace_back(input); } -static void basicDerivationToJson(json & res, const nix::BasicDerivation & d) +static void inputsToJson(json & res, const nix::FullInputs & inputs) +{ + using namespace nix; + res = nlohmann::json::object(); + + inputsToJson(res["srcs"], inputs.srcs); + + auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + + auto & inputDrvsObj = res["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputs.drvs.map) + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); +} + +template +void adl_serializer>::to_json(json & res, const nix::DerivationT & d) { using namespace nix; res = nlohmann::json::object(); @@ -1566,6 +1648,8 @@ static void basicDerivationToJson(json & res, const nix::BasicDerivation & d) outputsObj[outputName] = output; } + inputsToJson(res["inputs"], d.inputs); + res["system"] = d.platform; res["builder"] = d.builder; res["args"] = d.args; @@ -1575,157 +1659,119 @@ static void basicDerivationToJson(json & res, const nix::BasicDerivation & d) res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -void adl_serializer::to_json(json & res, const nix::BasicDerivation & d) -{ - basicDerivationToJson(res, d); +template +static Inputs inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings); - inputSrcsToJson(res["inputs"], d.inputSrcs); -} - -void adl_serializer::to_json(json & res, const nix::Derivation & d) +template<> +nix::StorePathSet inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings &) { using namespace nix; - - basicDerivationToJson(res, d); - - { - auto & inputsObj = res["inputs"]; - inputsObj = nlohmann::json::object(); - - inputSrcsToJson(inputsObj["srcs"], d.inputSrcs); - - auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { - auto value = nlohmann::json::object(); - value["outputs"] = inputNode.value; - { - auto next = nlohmann::json::object(); - for (auto & [outputId, childNode] : inputNode.childMap) - next[outputId] = doInput(childNode); - value["dynamicOutputs"] = std::move(next); - } - return value; - }; - - auto & inputDrvsObj = inputsObj["drvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } -} - -static void inputSrcsFromJson(const json & inputSrcsJson, nix::StorePathSet & inputSrcs) -{ - auto arr = nix::getArray(inputSrcsJson); - for (auto & input : arr) + StorePathSet inputSrcs; + for (auto & input : getArray(inputsJson)) inputSrcs.insert(input); + return inputSrcs; } -static void basicDerivationFromJson( - const json::object_t & json, nix::BasicDerivation & res, const nix::ExperimentalFeatureSettings & xpSettings) +template<> +nix::FullInputs +inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings) { using namespace nix; - res.name = getString(valueAt(json, "name")); - - { - auto version = getUnsigned(valueAt(json, "version")); - if (valueAt(json, "version") != expectedJsonVersionDerivation) - throw Error( - "Unsupported derivation JSON format version %d, only format version %d is currently supported.", - version, - expectedJsonVersionDerivation); - } + auto inputsObj = getObject(inputsJson); + FullInputs inputs; try { - auto outputs = getObject(valueAt(json, "outputs")); - for (auto & [outputName, output] : outputs) { - res.outputs.insert_or_assign(outputName, adl_serializer::from_json(output, xpSettings)); - } + for (auto & input : getArray(valueAt(inputsObj, "srcs"))) + inputs.srcs.insert(input); } catch (Error & e) { - e.addTrace({}, "while reading key 'outputs'"); + e.addTrace({}, "while reading key 'srcs'"); throw; } - res.platform = getString(valueAt(json, "system")); - res.builder = getString(valueAt(json, "builder")); - res.args = getStringList(valueAt(json, "args")); - - auto envJson = valueAt(json, "env"); try { - res.env = getStringMap(envJson); + auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { + xpSettings.require( + Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(inputsObj, "drvs"))) + inputs.drvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); + e.addTrace({}, "while reading key 'drvs'"); throw; } - if (auto structuredAttrs = get(json, "structuredAttrs")) - res.structuredAttrs = StructuredAttrs{*structuredAttrs}; + return inputs; } -nix::BasicDerivation -adl_serializer::from_json(const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) +template +nix::DerivationT adl_serializer>::from_json( + const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) { using namespace nix; - BasicDerivation res; auto & json = getObject(_json); - basicDerivationFromJson(json, res, xpSettings); - - try { - inputSrcsFromJson(valueAt(json, "inputs"), res.inputSrcs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputs'"); - throw; + { + auto version = getUnsigned(valueAt(json, "version")); + if (version != expectedJsonVersionDerivation) + throw Error( + "Unsupported derivation JSON format version %d, only format version %d is currently supported.", + version, + expectedJsonVersionDerivation); } - return res; -} - -nix::Derivation -adl_serializer::from_json(const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) -{ - using namespace nix; - - Derivation res; - auto & json = getObject(_json); - basicDerivationFromJson(json, res, xpSettings); - - try { - auto inputsObj = getObject(valueAt(json, "inputs")); - - try { - inputSrcsFromJson(valueAt(inputsObj, "srcs"), res.inputSrcs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'srcs'"); - throw; - } - - try { - auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { - auto & json = getObject(_json); - DerivedPathMap::ChildNode node; - node.value = getStringSet(valueAt(json, "outputs")); - auto drvs = getObject(valueAt(json, "dynamicOutputs")); - for (auto & [outputId, childNode] : drvs) { - xpSettings.require( - Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); - node.childMap[outputId] = doInput(childNode); + return DerivationT{ + .outputs = + [&] { + DerivationOutputs outputs; + try { + for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) + outputs.insert_or_assign( + outputName, adl_serializer::from_json(output, xpSettings)); + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; } - return node; - }; - auto drvs = getObject(valueAt(inputsObj, "drvs")); - for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'drvs'"); - throw; - } - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputs'"); - throw; - } - - return res; + return outputs; + }(), + .inputs = + [&] { + try { + return inputsFromJson(valueAt(json, "inputs"), xpSettings); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + }(), + .platform = getString(valueAt(json, "system")), + .builder = getString(valueAt(json, "builder")), + .args = getStringList(valueAt(json, "args")), + .env = + [&] { + try { + return getStringMap(valueAt(json, "env")); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + }(), + .structuredAttrs = [&]() -> std::optional { + if (auto structuredAttrs = get(json, "structuredAttrs")) + return StructuredAttrs{*structuredAttrs}; + return std::nullopt; + }(), + .name = getString(valueAt(json, "name")), + }; } +template struct adl_serializer; +template struct adl_serializer; + } // namespace nlohmann diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5e62408847bf..0bbd4b9f09cd 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -279,7 +279,7 @@ bool Settings::isWSL1() #endif } -const ExternalBuilder * LocalSettings::findExternalDerivationBuilderIfSupported(const Derivation & drv) +const ExternalBuilder * LocalSettings::findExternalDerivationBuilderIfSupported(const BasicDerivation & drv) { if (auto it = std::ranges::find_if( externalBuilders.get(), [&](const auto & handler) { return handler.systems.contains(drv.platform); }); diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 6a17f73eeb51..08b3fa80d4b2 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -34,20 +34,11 @@ struct DerivationBuildingGoal : public Goal friend class Worker; /** - * @param storeDerivation Whether to store the derivation in - * `worker.store`. This is useful for newly-resolved derivations. In this - * case, the derivation was not created a priori, e.g. purely (or close - * enough) from evaluation of the Nix language, but also depends on the - * exact content produced by upstream builds. It is strongly advised to - * have a permanent record of such a resolved derivation in order to - * faithfully reconstruct the build history. + * @param drv The derivation to build, with the outputs of its input + * derivations already added to its input sources. */ DerivationBuildingGoal( - const StorePath & drvPath, - ref drv, - Worker & worker, - BuildMode buildMode, - bool storeDerivation); + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); ~DerivationBuildingGoal(); private: @@ -56,9 +47,9 @@ private: const StorePath drvPath; /** - * The derivation stored at drvPath. + * The derivation to build. */ - const ref drv; + const ref drv; /** * The remainder is state held during the build. @@ -79,7 +70,7 @@ private: /** * The states. */ - Co gaveUpOnSubstitution(bool storeDerivation); + Co gaveUpOnSubstitution(); Co tryToBuild(StorePathSet inputPaths); Co buildWithHook( StorePathSet inputPaths, diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 8d6892839c76..dfdc25d0ed1a 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -9,7 +9,11 @@ namespace nix { class Store; -struct Derivation; + +template +struct DerivationT; +struct FullInputs; +using Derivation = DerivationT; /** * Unless we are repairing, we don't both to test validity and just assume it, @@ -51,6 +55,7 @@ struct InitialOutput /** * Format the known outputs of a derivation for use in error messages. */ -std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv); +template +std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv); } // namespace nix diff --git a/src/libstore/include/nix/store/build/derivation-env-desugar.hh b/src/libstore/include/nix/store/build/derivation-env-desugar.hh index a10ec9fa8736..ce65c7cbe8cf 100644 --- a/src/libstore/include/nix/store/build/derivation-env-desugar.hh +++ b/src/libstore/include/nix/store/build/derivation-env-desugar.hh @@ -7,7 +7,13 @@ namespace nix { class Store; -struct Derivation; + +template +struct DerivationT; +struct FullInputs; +using Derivation = DerivationT; +using BasicDerivation = DerivationT; + template struct DerivationOptions; @@ -79,7 +85,7 @@ struct DesugaredEnv */ static DesugaredEnv create( Store & store, - const Derivation & drv, + const BasicDerivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths); }; diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index d8bbfa1ef5c4..9af1997e34dd 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -306,8 +306,8 @@ public: /** * @ref DerivationBuildingGoal "derivation building goal" */ - std::shared_ptr makeDerivationBuildingGoal( - const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation); + std::shared_ptr + makeDerivationBuildingGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref PathSubstitutionGoal "substitution goal" diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index e29f660c4848..0931b41143f7 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -13,8 +13,15 @@ namespace nix { +class Store; + struct StoreDirConfig; -struct BasicDerivation; + +template +struct DerivationT; +struct FullInputs; +using BasicDerivation = DerivationT; + struct StructuredAttrs; template @@ -180,14 +187,16 @@ struct DerivationOptions * the future we'll flip things around so a `BasicDerivation` has * `DerivationOptions` instead. */ - StringSet getRequiredSystemFeatures(const BasicDerivation & drv) const; + template + StringSet getRequiredSystemFeatures(const DerivationT & drv) const; bool substitutesAllowed(const WorkerSettings & workerSettings) const; /** * @param drv See note on `getRequiredSystemFeatures` */ - bool useUidRange(const BasicDerivation & drv) const; + template + bool useUidRange(const DerivationT & drv) const; }; extern template struct DerivationOptions; diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 6d860ee1840c..369b1ca30e2d 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -152,6 +152,23 @@ typedef std::map DerivationInputs; +/** + * Inputs for full Derivation - both source and derivation inputs + */ +struct FullInputs +{ + /** + * inputs that are sources + */ + StorePathSet srcs; + /** + * inputs that are sub-derivations + */ + DerivedPathMap>> drvs; + + bool operator==(const FullInputs &) const = default; +}; + struct DerivationType { /** @@ -263,16 +280,20 @@ struct DerivationType bool hasKnownOutputPaths() const; }; -struct BasicDerivation +template +struct DerivationT; + +using BasicDerivation = DerivationT; +using Derivation = DerivationT; + +template +struct DerivationT { /** * keyed on symbolic IDs */ DerivationOutputs outputs; - /** - * inputs that are sources - */ - StorePathSet inputSrcs; + Inputs inputs; std::string platform; /** * Probably should be an absolute path in the path format that `platform` uses @@ -287,12 +308,7 @@ struct BasicDerivation std::string name; - BasicDerivation() = default; - BasicDerivation(BasicDerivation &&) = default; - BasicDerivation(const BasicDerivation &) = default; - BasicDerivation & operator=(BasicDerivation &&) = default; - BasicDerivation & operator=(const BasicDerivation &) = default; - virtual ~BasicDerivation(); + bool operator==(const DerivationT &) const = default; bool isBuiltin() const; @@ -321,27 +337,14 @@ struct BasicDerivation */ void applyRewrites(const StringMap & rewrites); - bool operator==(const BasicDerivation &) const = default; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - // auto operator <=> (const BasicDerivation &) const = default; -}; - -class Store; - -struct Derivation : BasicDerivation -{ - /** - * inputs that are sub-derivations - */ - DerivedPathMap>> inputDrvs; - /** - * Print a derivation. + * Print a derivation (only meaningful for full Derivation). */ std::string unparse( const StoreDirConfig & store, bool maskOutputs, - DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; + DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const + requires std::is_same_v; /** * Determine whether this derivation should be resolved before building. @@ -354,7 +357,8 @@ struct Derivation : BasicDerivation * - Impure derivations always need resolution * - Any input derivations have outputs from dynamic derivations */ - bool shouldResolve() const; + bool shouldResolve() const + requires std::is_same_v; /** * Return the underlying basic derivation but with these changes: @@ -365,7 +369,8 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const + requires std::is_same_v; /** * Like the above, but instead of querying the Nix database for @@ -375,7 +380,34 @@ struct Derivation : BasicDerivation std::optional tryResolve( Store & store, fun(ref drvPath, const std::string & outputName)> - queryResolutionChain) const; + queryResolutionChain) const + requires std::is_same_v; + + /** + * Convert a BasicDerivation to a full Derivation. + * The resulting Derivation has empty inputDrvs since BasicDerivation + * is already resolved. + */ + Derivation unresolve() const + requires std::is_same_v; + + /** + * Return a derivation identical to this one, but with the inputs transformed by `f`. + */ + template + DerivationT> mapInputs(F f) const + { + return { + .outputs = outputs, + .inputs = f(inputs), + .platform = platform, + .builder = builder, + .args = args, + .env = env, + .structuredAttrs = structuredAttrs, + .name = name, + }; + } /** * Check that the derivation is valid and does not present any @@ -424,24 +456,8 @@ struct Derivation : BasicDerivation * @param store The store to use for path computation * @param drvName The derivation name (without .drv extension) */ - void fillInOutputPaths(Store & store); - - Derivation() = default; - Derivation(Derivation &&) = default; - Derivation(const Derivation &) = default; - Derivation & operator=(Derivation &&) = default; - Derivation & operator=(const Derivation &) = default; - ~Derivation() override; - - Derivation(const BasicDerivation & bd) - : BasicDerivation(bd) - { - } - - Derivation(BasicDerivation && bd) - : BasicDerivation(std::move(bd)) - { - } + void fillInOutputPaths(Store & store) + requires std::is_same_v; /** * Parse a derivation from JSON, and also perform various @@ -464,15 +480,35 @@ struct Derivation : BasicDerivation * @return A validated derivation with output paths filled in * @throws Error if parsing fails, output paths can't be computed, or validation fails */ - static Derivation parseJsonAndValidate(Store & store, const nlohmann::json & json); - - bool operator==(const Derivation &) const = default; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - // auto operator <=> (const Derivation &) const = default; + static Derivation parseJsonAndValidate(Store & store, const nlohmann::json & json) + requires std::is_same_v; }; class Store; +template<> +std::string DerivationT::unparse( + const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const; +template<> +bool DerivationT::shouldResolve() const; +template<> +std::optional DerivationT::tryResolve(Store & store, Store * evalStore) const; +template<> +std::optional DerivationT::tryResolve( + Store & store, + fun(ref drvPath, const std::string & outputName)> + queryResolutionChain) const; +template<> +void DerivationT::fillInOutputPaths(Store & store); +template<> +Derivation DerivationT::parseJsonAndValidate(Store & store, const nlohmann::json & json); +template<> +Derivation DerivationT::unresolve() const; +template<> +void DerivationT::checkInvariants(Store & store) const; +template<> +void DerivationT::checkInvariants(Store & store) const; + /** * Compute the store path that would be used for a derivation without writing it. * @@ -625,5 +661,8 @@ constexpr unsigned expectedJsonVersionDerivation = 4; } // namespace nix JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) -JSON_IMPL_WITH_XP_FEATURES(nix::BasicDerivation) -JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) + +namespace nlohmann { +template +JSON_IMPL_WITH_XP_FEATURES_INNER(nix::DerivationT); +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/downstream-placeholder.hh b/src/libstore/include/nix/store/downstream-placeholder.hh index ba3e9faeff70..6fe252bf938d 100644 --- a/src/libstore/include/nix/store/downstream-placeholder.hh +++ b/src/libstore/include/nix/store/downstream-placeholder.hh @@ -37,7 +37,7 @@ using DrvRef = std::variant; * We use them with `Derivation`: the `render()` method is called to * render an opaque string which can be used in the derivation, and the * resolving logic can substitute those strings for store paths when - * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`. + * resolving `Derivation.inputs.drvs` to `BasicDerivation.inputs.srcs`. */ class DownstreamPlaceholder { diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 8e4833591cfa..46186fd6ae1b 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -736,7 +736,7 @@ public: * Finds the first external derivation builder that supports this * derivation, or else returns a null pointer. */ - const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); + const ExternalBuilder * findExternalDerivationBuilderIfSupported(const BasicDerivation & drv); }; template<> diff --git a/src/libstore/include/nix/store/outputs-query.hh b/src/libstore/include/nix/store/outputs-query.hh index 62a2524ddb60..0129688acd2a 100644 --- a/src/libstore/include/nix/store/outputs-query.hh +++ b/src/libstore/include/nix/store/outputs-query.hh @@ -19,7 +19,7 @@ using QueryRealisationFun = std::function> & outputs, QueryRealisationFun queryRealisation = {}); diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 1e77ecfe5ccf..cd512f4c0f74 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -36,8 +36,11 @@ struct Realisation; struct RealisedPath; struct DrvOutput; -struct BasicDerivation; -struct Derivation; +template +struct DerivationT; +struct FullInputs; +using BasicDerivation = DerivationT; +using Derivation = DerivationT; struct SourceAccessor; struct NarInfoDiskCache; @@ -1107,7 +1110,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven = std::nullopt); -const ContentAddress * getDerivationCA(const BasicDerivation & drv); +const ContentAddress * getDerivationCA(const Derivation & drv); template<> struct json_avoids_null : std::true_type diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 60ec3fb31e0f..02ba90ec17b6 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -75,7 +75,7 @@ void Store::computeFSClosure( computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } -const ContentAddress * getDerivationCA(const BasicDerivation & drv) +const ContentAddress * getDerivationCA(const Derivation & drv) { auto out = drv.outputs.find("out"); if (out == drv.outputs.end()) @@ -184,7 +184,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv, std::set & edges) { res.willBuild.insert(drvPath); - for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + for (const auto & [inputDrv, inputNode] : drv.inputs.drvs.map) collectDerivedPaths(edges, makeConstantStorePathRef(inputDrv), inputNode); }; @@ -231,7 +231,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) // FIXME: this is a lot of work just to get the value // of `allowSubstitutes`. drvOptions = derivationOptionsFromStructuredAttrs( - *this, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); + *this, drv->inputs.drvs, drv->env, get(drv->structuredAttrs)); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); throw; diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index 6a5cce222339..5d34596482f3 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -105,7 +105,7 @@ static std::pair resolveDerivation( store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, cache, resCache); }); if (resolvedDrv) - drv = Derivation{*resolvedDrv}; + drv = resolvedDrv->unresolve(); } auto resolvedDrvPath = computeStorePath(store, drv); @@ -117,7 +117,7 @@ static std::pair resolveDerivation( void queryPartialDerivationOutputMapCA( Store & store, const StorePath & drvPath, - const BasicDerivation & drv, + const Derivation & drv, std::map> & outputs, QueryRealisationFun queryRealisation, RealisationCache & resCache) @@ -185,7 +185,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( void queryPartialDerivationOutputMapCA( Store & store, const StorePath & drvPath, - const BasicDerivation & drv, + const Derivation & drv, std::map> & outputs, QueryRealisationFun queryRealisation) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c6809a248fa6..e1d7493f7b67 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1230,7 +1230,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) // resolved derivation, so we need to get it first auto resolvedDrv = drv.tryResolve(*this); if (resolvedDrv) - return nix::computeStorePath(*this, Derivation{*resolvedDrv}); + return nix::computeStorePath(*this, resolvedDrv->unresolve()); } return path; diff --git a/src/libutil/include/nix/util/json-impls.hh b/src/libutil/include/nix/util/json-impls.hh index 5a1c1354309a..05c299cd535e 100644 --- a/src/libutil/include/nix/util/json-impls.hh +++ b/src/libutil/include/nix/util/json-impls.hh @@ -31,14 +31,17 @@ JSON_IMPL_INNER(TYPE); \ } -#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \ - namespace nlohmann { \ - template<> \ +#define JSON_IMPL_WITH_XP_FEATURES_INNER(TYPE) \ struct adl_serializer \ { \ static TYPE from_json( \ const json & json, \ const nix::ExperimentalFeatureSettings & xpSettings = nix::experimentalFeatureSettings); \ static void to_json(json & json, const TYPE & t); \ - }; \ + } + +#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \ + namespace nlohmann { \ + template<> \ + JSON_IMPL_WITH_XP_FEATURES_INNER(TYPE); \ } diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 648f05a489c1..19d4ab095e61 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -328,18 +328,26 @@ static int main_build_remote(int argc, char ** argv) // This condition mirrors that: that code enforces the "rules" outlined there; // we do the best we can given those "rules". if (trustedOrLegacy || drv.type().isCA()) { - // Hijack the inputs paths of the derivation to include all - // the paths that come from the `inputDrvs` set. We don’t do - // that for the derivations whose `inputDrvs` is empty - // because: - // - // 1. It’s not needed - // - // 2. Changing the `inputSrcs` set changes the associated - // output ids, which break CA derivations - if (!drv.inputDrvs.map.empty()) - drv.inputSrcs = store->parseStorePathSet(inputs); - optResult = sshStore->getBuilder()->buildDerivation(*drvPath, static_cast(drv)); + BasicDerivation resolvedDrv{ + .outputs = drv.outputs, + // Hijack the inputs paths of the derivation to include + // all the paths that come from the `inputDrvs` set. We + // don’t do that for the derivations whose `inputDrvs` + // is empty because: + // + // 1. It’s not needed + // + // 2. Changing the `inputSrcs` set changes the + // associated output ids, which break CA derivations + .inputs = drv.inputs.drvs.map.empty() ? drv.inputs.srcs : store->parseStorePathSet(inputs), + .platform = drv.platform, + .builder = drv.builder, + .args = drv.args, + .env = drv.env, + .structuredAttrs = drv.structuredAttrs, + .name = drv.name, + }; + optResult = sshStore->getBuilder()->buildDerivation(*drvPath, resolvedDrv); auto & result = *optResult; if (auto * failureP = result.tryGetFailure()) { if (settings.keepFailed) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1243738af456..169c050e89e2 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -268,7 +268,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore 'buildDerivation', but that's privileged. */ drv.name += "-env"; drv.env.emplace("name", drv.name); - drv.inputSrcs.insert(std::move(getEnvShPath)); + drv.inputs.srcs.insert(std::move(getEnvShPath)); for (auto & [outputName, output] : drv.outputs) { std::visit( overloaded{ diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index b7c9531b1481..f359f7d9ae1e 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -511,7 +511,7 @@ static void main_nix_build(int argc, char ** argv) }; // Build or fetch all dependencies of the derivation. - for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) { + for (const auto & [inputDrv0, inputNode] : drv.inputs.drvs.map) { // To get around lambda capturing restrictions in the // standard. const auto & inputDrv = inputDrv0; @@ -522,7 +522,7 @@ static void main_nix_build(int argc, char ** argv) pathsToCopy.insert(inputDrv); } } - for (const auto & src : drv.inputSrcs) { + for (const auto & src : drv.inputs.srcs) { pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } @@ -545,7 +545,7 @@ static void main_nix_build(int argc, char ** argv) auto resolvedDrv = drv.tryResolve(*store); if (!resolvedDrv) throw Error("failed to resolve derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); - drv = *resolvedDrv; + drv = resolvedDrv->unresolve(); } // Set the environment. @@ -611,7 +611,7 @@ static void main_nix_build(int argc, char ** argv) } }; - for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + for (const auto & [inputDrv, inputNode] : drv.inputs.drvs.map) accumInputClosure(inputDrv, inputNode); auto json = drv.structuredAttrs->prepareStructuredAttrs(*store, drvOptions, inputs, drv.outputs); From 1dc97c43074028aea9f403373d0b7bd0df242749 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 7 Jun 2026 13:30:25 -0400 Subject: [PATCH 355/364] Use designated initializers for derivations in more places This is much nicer to read than imperatively mutating fields one by one. Assisted-by: Claude:opus-4.8 --- src/libexpr/primops.cc | 5 +- .../derivation/external-formats.cc | 144 +++++++----------- src/libstore-tests/derivation/invariants.cc | 140 +++++++---------- src/libstore-tests/derivations.cc | 101 ++++++------ src/libstore-tests/dummy-store.cc | 4 +- src/libstore-tests/outputs-query.cc | 12 +- .../register-valid-paths-bench.cc | 13 +- src/libstore-tests/worker-substitution.cc | 42 ++--- src/libstore-tests/write-derivation.cc | 16 +- src/libstore/derivations.cc | 5 +- 10 files changed, 216 insertions(+), 266 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2428d98768a1..6bf8b4ad271c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1515,8 +1515,9 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ - Derivation drv; - drv.name = drvName; + Derivation drv{ + .name = std::string{drvName}, + }; NixStringContext context; diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index d1ccc203c97a..412d192ced62 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -74,11 +74,10 @@ INSTANTIATE_TEST_SUITE_P( std::pair{ "caFixedNAR", DerivationOutput{DerivationOutput::CAFixed{ - .ca = - { - .method = ContentAddressMethod::Raw::NixArchive, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, + .ca{ + .method = ContentAddressMethod::Raw::NixArchive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, }}, }, std::pair{ @@ -185,41 +184,36 @@ struct DerivationJsonAtermTest : DerivationTest, MAKE_TEST_P(DerivationJsonAtermTest); -INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { - Derivation drv; - drv.name = "simple-derivation"; - drv.inputs.srcs = { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputs.drvs = { - .map = - { - { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = - { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; - }())); +INSTANTIATE_TEST_SUITE_P( + DerivationJSONATerm, + DerivationJsonAtermTest, + ::testing::Values( + Derivation{ + .outputs = {}, + .inputs{ + .srcs{ + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, + }, + .drvs{.map{ + { + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"}, + { + .value{ + "cat", + "dog", + }, + }, + }, + }}, + }, + .platform = "wasm-sel4", + .builder = "foo", + .args = {"bar", "baz"}, + .env{ + {"BIG_BAD", "WOLF"}, + }, + .name = "simple-derivation", + })); struct DynDerivationJsonAtermTest : DynDerivationTest, JsonCharacterizationTest, @@ -230,60 +224,36 @@ MAKE_TEST_P(DynDerivationJsonAtermTest); Derivation makeDynDepDerivation() { - Derivation drv; - drv.name = "dyn-dep-derivation"; - drv.inputs.srcs = { - StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, - }; - drv.inputs.drvs = { - .map = - { + return Derivation{ + .outputs = {}, + .inputs{ + .srcs{ + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, + }, + .drvs{.map{ { StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"}, DerivedPathMap::ChildNode{ - .value = - { - "cat", - "dog", - }, - .childMap = - { - { - "cat", - DerivedPathMap::ChildNode{ - .value = - { - "kitten", - }, - }, - }, - { - "goose", - DerivedPathMap::ChildNode{ - .value = - { - "gosling", - }, - }, - }, - }, + .value{ + "cat", + "dog", + }, + .childMap{ + {"cat", DerivedPathMap::ChildNode{.value = {"kitten"}}}, + {"goose", DerivedPathMap::ChildNode{.value = {"gosling"}}}, + }, }, }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", + }}, + }, + .platform = "wasm-sel4", + .builder = "foo", + .args = {"bar", "baz"}, + .env{ + {"BIG_BAD", "WOLF"}, }, + .name = "dyn-dep-derivation", }; - return drv; } INSTANTIATE_TEST_SUITE_P(DynDerivationJSONATerm, DynDerivationJsonAtermTest, ::testing::Values(makeDynDepDerivation())); diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index edffe176d040..eea26b78e99a 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -29,21 +29,21 @@ class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTe */ StorePath makeCAFloatingDependency(std::string_view name) { - Derivation depDrv; - depDrv.name = name; - depDrv.platform = "x86_64-linux"; - depDrv.builder = "/bin/sh"; - depDrv.outputs = { - { - "out", - // will ensure that downstream is deferred - DerivationOutput{DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }}, + Derivation depDrv{ + .outputs{ + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, }, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"out", ""}}, + .name = std::string{name}, }; - depDrv.env = {{"out", ""}}; // Fill in the dependency derivation's output paths depDrv.fillInOutputPaths(*store); @@ -64,14 +64,13 @@ TEST_F(FillInOutputPathsTest, fillsDeferredOutputs_emptyStringEnvVar) using nlohmann::json; // Before: Derivation with deferred output - Derivation drv; - drv.name = "filled-in-deferred-empty-env-var"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Fill in deferred output with empty env var"}, {"out", ""}}, + .name = "filled-in-deferred-empty-env-var", }; - drv.env = {{"__doc", "Fill in deferred output with empty env var"}, {"out", ""}}; // Serialize before state checkpointJson("filled-in-deferred-empty-env-var-pre", drv); @@ -95,15 +94,12 @@ TEST_F(FillInOutputPathsTest, fillsDeferredOutputs_empty_string_var) using nlohmann::json; // Before: Derivation with deferred output - Derivation drv; - drv.name = "filled-in-deferred-no-env-var"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, - }; - drv.env = { - {"__doc", "Fill in deferred with missing env var"}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Fill in deferred with missing env var"}}, + .name = "filled-in-deferred-no-env-var", }; // Serialize before state @@ -127,16 +123,12 @@ TEST_F(FillInOutputPathsTest, preservesInputAddressedOutputs) { auto expectedPath = StorePath{"w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"}; - Derivation drv; - drv.name = "filled-in-already"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::InputAddressed{.path = expectedPath}}}, - }; - drv.env = { - {"__doc", "Correct path stays unchanged"}, - {"out", store->printStorePath(expectedPath)}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = expectedPath}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Correct path stays unchanged"}, {"out", store->printStorePath(expectedPath)}}, + .name = "filled-in-already", }; // Serialize before state @@ -154,16 +146,12 @@ TEST_F(FillInOutputPathsTest, throwsOnIncorrectInputAddressedPath) { auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"}; - Derivation drv; - drv.name = "bad-path"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}, - }; - drv.env = { - {"__doc", "Wrong InputAddressed path throws error"}, - {"out", store->printStorePath(wrongPath)}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Wrong InputAddressed path throws error"}, {"out", store->printStorePath(wrongPath)}}, + .name = "bad-path", }; // Serialize before state @@ -177,16 +165,12 @@ TEST_F(FillInOutputPathsTest, throwsOnIncorrectEnvVar) { auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"}; - Derivation drv; - drv.name = "bad-env-var"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, - }; - drv.env = { - {"__doc", "Wrong env var value throws error"}, - {"out", store->printStorePath(wrongPath)}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Wrong env var value throws error"}, {"out", store->printStorePath(wrongPath)}}, + .name = "bad-env-var", }; // Serialize before state @@ -204,19 +188,14 @@ TEST_F(FillInOutputPathsTest, preservesDeferredWithInputDrvs) auto depDrvPath = makeCAFloatingDependency("dependency"); // Create a derivation that depends on the dependency - Derivation drv; - drv.name = "depends-on-drv"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, - }; - drv.env = { - {"__doc", "Deferred stays deferred with CA dependencies"}, - {"out", ""}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Deferred stays deferred with CA dependencies"}, {"out", ""}}, + .name = "depends-on-drv", }; - // Add the real input derivation dependency - drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Serialize before state checkpointJson("depends-on-drv-pre", drv); @@ -240,19 +219,14 @@ TEST_F(FillInOutputPathsTest, throwsOnPatWhenShouldBeDeffered) auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"}; // Create a derivation that depends on the dependency - Derivation drv; - drv.name = "depends-on-drv"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}, - }; - drv.env = { - {"__doc", "InputAddressed throws when should be deferred"}, - {"out", ""}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "InputAddressed throws when should be deferred"}, {"out", ""}}, + .name = "depends-on-drv", }; - // Add the real input derivation dependency - drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Serialize before state checkpointJson("bad-depends-on-drv-pre", drv); diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc index 39449baf9128..03a67a8795f5 100644 --- a/src/libstore-tests/derivations.cc +++ b/src/libstore-tests/derivations.cc @@ -128,13 +128,13 @@ TEST_F(TryResolveTest, noInputs) resolveExpect( "no-inputs", [&] { - Derivation drv; - drv.name = "no-inputs"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.env = {{"FOO", "bar"}}; - return drv; + return Derivation{ + .outputs = {{"out", caFloatingOutput()}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .env = {{"FOO", "bar"}}, + .name = "no-inputs", + }; }(), {}, [&] { @@ -168,28 +168,28 @@ TEST_F(TryResolveTest, withInputs) resolveExpect( "with-inputs", - [&] { - Derivation drv; - drv.name = "with-inputs"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = multiOutputs; - drv.inputs.drvs = { - .map = { + Derivation{ + .outputs = multiOutputs, + .inputs{ + .drvs{.map{ {dep1DrvPath, {.value = {"out", "dev"}}}, {dep2DrvPath, {.value = {"out"}}}, - }}; - drv.env = { - {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, - {"DEP1_DEV", placeholder1Dev}, - {"DEP2", placeholder2Out}, - }; - drv.structuredAttrs = StructuredAttrs{{ + }}, + }, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .env = + { + {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, + {"DEP1_DEV", placeholder1Dev}, + {"DEP2", placeholder2Out}, + }, + .structuredAttrs = StructuredAttrs{{ {"dep1out", placeholder1Out}, {"nested", nlohmann::json::object({{"dep2", "before " + placeholder2Out + " after"}})}, - }}; - return drv; - }(), + }}, + .name = "with-inputs", + }, {.dict{ { SingleDerivedPath::Built{ @@ -238,12 +238,13 @@ TEST_F(TryResolveTest, resolutionFailure) { StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; - Derivation drv; - drv.name = "resolution-failure"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + Derivation drv{ + .outputs = {{"out", caFloatingOutput()}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .name = "resolution-failure", + }; BuildTrace buildTrace; @@ -328,14 +329,13 @@ TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); - Derivation drv; - drv.name = "export-ref-subpath"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; - drv.env = { - {"exportReferencesGraph", "refs " + placeholder + "/foo"}, + Derivation drv{ + .outputs = {{"out", caFloatingOutput()}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .env = {{"exportReferencesGraph", "refs " + placeholder + "/foo"}}, + .name = "export-ref-subpath", }; exportRefGraphSubpathTest("export-ref-subpath", drv, nullptr); @@ -346,15 +346,20 @@ TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); - Derivation drv; - drv.name = "export-ref-subpath-sa"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; - drv.structuredAttrs = StructuredAttrs{{ - {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, - }}; + Derivation drv{ + .outputs = {{"out", caFloatingOutput()}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .structuredAttrs = StructuredAttrs{{ + { + "exportReferencesGraph", + nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}}), + }, + }}, + .name = "export-ref-subpath-sa", + }; + // env depends on structuredAttrs, so set it after construction drv.env = { {std::string{StructuredAttrs::envVarName}, nlohmann::json(drv.structuredAttrs->structuredAttrs).dump()}, }; diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 4626fc31f28d..6c2625d5306a 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -158,9 +158,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { "one-derivation", [&] { auto store = writeCfg->openDummyStore(); - Derivation drv; - drv.name = "foo"; - store->writeDerivation(drv); + store->writeDerivation(Derivation{.name = "foo"}); return store; }(), }, diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc index de585af61987..356df632b8da 100644 --- a/src/libstore-tests/outputs-query.cc +++ b/src/libstore-tests/outputs-query.cc @@ -43,12 +43,12 @@ class OutputsQueryTest : public ::testing::Test */ Derivation makeLeafDrv(std::string name) { - Derivation drv; - drv.name = std::move(name); - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = {{"out", caFloatingOutput()}}; - return drv; + return Derivation{ + .outputs = {{"out", caFloatingOutput()}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .name = std::move(name), + }; } }; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index ecea1c8010a4..5126f243a5b2 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -34,12 +34,13 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) std::string drvName = fmt("register-valid-paths-bench-%d", i); auto drvPath = StorePath::random(drvName + ".drv"); - Derivation drv; - drv.name = drvName; - drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}}); - drv.platform = "x86_64-linux"; - drv.builder = "foo"; - drv.env["out"] = ""; + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "foo", + .env = {{"out", ""}}, + .name = drvName, + }; drv.fillInOutputPaths(*localStore); auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false); diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 2a19cf7580a3..29a7e3a8320d 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -179,16 +179,17 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) EnableExperimentalFeature enableCA{"ca-derivations"}; // Create a CA floating output derivation - Derivation drv; - drv.name = "test-ca-drv"; - drv.outputs = { - { - "out", - DerivationOutput{DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }}, + Derivation drv{ + .outputs{ + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, }, + .name = "test-ca-drv", }; // Write the derivation to the destination store @@ -317,19 +318,20 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) }); // Create the root CA floating derivation that depends on depDrv - Derivation rootDrv; - rootDrv.name = "root-drv"; - rootDrv.outputs = { - { - "out", - DerivationOutput{DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }}, + Derivation rootDrv{ + .outputs{ + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, }, + // Add the dependency derivation as an input + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .name = "root-drv", }; - // Add the dependency derivation as an input - rootDrv.inputs.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Write the root derivation to the destination store auto rootDrvPath = dummyStore->writeDerivation(rootDrv); diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index 43060cb697b8..65b475387b54 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -31,15 +31,13 @@ class WriteDerivationTest : public LibStoreTest TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { - auto drv = []() { - Derivation drv; - drv.name = "simple-derivation"; - drv.platform = "system"; - drv.builder = "foo"; - drv.args = {"bar", "baz"}; - drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; - return drv; - }(); + Derivation drv{ + .platform = "system", + .builder = "foo", + .args = {"bar", "baz"}, + .env = {{"BIG_BAD", "WOLF"}}, + .name = "simple-derivation", + }; auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 17f2285b33e6..dd7d69585c6d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -451,8 +451,9 @@ Derivation parseDerivation( { using namespace std::literals::string_view_literals; - Derivation drv; - drv.name = name; + Derivation drv{ + .name = std::string{name}, + }; StringViewStream str{s}; expect(str, 'D'); From c9a84e2dc381fc76c9c3c4eb506a305241c8e7c9 Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Tue, 21 Jul 2026 13:29:47 -0400 Subject: [PATCH 356/364] libexpr: Give better warning messages for derivation attributes The previous code had excessive message duplication yet did not include the line of the attribute. Replace with a new `warnAttr` closure to simplify. Co-Authored-By: John Ericson --- src/libexpr/primops.cc | 42 +++++-------- .../lang/eval-okay-derivation-legacy.err.exp | 60 +++++++++++++++++-- tests/functional/structured-attrs.sh | 2 +- 3 files changed, 71 insertions(+), 33 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6bf8b4ad271c..10023079f51c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1536,6 +1536,17 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); + // Like `warn`, but with the position of the attribute and the derivation name as an added trace. + auto warnAttr = [&](HintFmt msg) { + ErrorInfo info{ + .level = lvlWarn, + .msg = std::move(msg), + .pos = state.positions[i->pos], + }; + info.traces.push_back(Trace{.hint = HintFmt{"while evaluating derivation '%1%'", drvName}}); + logWarning(info); + }; + auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" @@ -1651,34 +1662,14 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName switch (i->name.getId()) { case EvalState::s.allowedReferences.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", - drvName); - break; case EvalState::s.allowedRequisites.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", - drvName); - break; case EvalState::s.disallowedReferences.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", - drvName); - break; case EvalState::s.disallowedRequisites.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", - drvName); - break; case EvalState::s.maxSize.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", - drvName); - break; case EvalState::s.maxClosureSize.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", - drvName); + warnAttr(HintFmt( + "'structuredAttrs' disables the effect of the derivation attribute '%1%'; use 'outputChecks..%1%' instead", + key)); break; default: break; @@ -1687,9 +1678,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); if (i->name == state.s.json) { - warn( - "In derivation '%s': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.", - drvName); + warnAttr(HintFmt( + "setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.")); drv.structuredAttrs = StructuredAttrs::parse(s); } else { drv.env.emplace(key, s); diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp index 94f0854dd2c9..fb0817fd5483 100644 --- a/tests/functional/lang/eval-okay-derivation-legacy.err.exp +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -1,6 +1,54 @@ -warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead -warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead -warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead -warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead -warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead -warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead +warning: + … while evaluating derivation 'eval-okay-derivation-legacy' + + warning: 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead + at /pwd/lang/eval-okay-derivation-legacy.nix:6:3: + 5| __structuredAttrs = true; + 6| allowedReferences = [ ]; + | ^ + 7| disallowedReferences = [ ]; +warning: + … while evaluating derivation 'eval-okay-derivation-legacy' + + warning: 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead + at /pwd/lang/eval-okay-derivation-legacy.nix:8:3: + 7| disallowedReferences = [ ]; + 8| allowedRequisites = [ ]; + | ^ + 9| disallowedRequisites = [ ]; +warning: + … while evaluating derivation 'eval-okay-derivation-legacy' + + warning: 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead + at /pwd/lang/eval-okay-derivation-legacy.nix:7:3: + 6| allowedReferences = [ ]; + 7| disallowedReferences = [ ]; + | ^ + 8| allowedRequisites = [ ]; +warning: + … while evaluating derivation 'eval-okay-derivation-legacy' + + warning: 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead + at /pwd/lang/eval-okay-derivation-legacy.nix:9:3: + 8| allowedRequisites = [ ]; + 9| disallowedRequisites = [ ]; + | ^ + 10| maxSize = 1234; +warning: + … while evaluating derivation 'eval-okay-derivation-legacy' + + warning: 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead + at /pwd/lang/eval-okay-derivation-legacy.nix:11:3: + 10| maxSize = 1234; + 11| maxClosureSize = 12345; + | ^ + 12| }).out +warning: + … while evaluating derivation 'eval-okay-derivation-legacy' + + warning: 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead + at /pwd/lang/eval-okay-derivation-legacy.nix:10:3: + 9| disallowedRequisites = [ ]; + 10| maxSize = 1234; + | ^ + 11| maxClosureSize = 12345; diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 3ea4eb18387a..5ba7547a1042 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -41,7 +41,7 @@ test "$(<<<"$jsonOut" jq '.variables.outputs.value.out' -r)" = "$(<<<"$jsonOut" hackyExpr='derivation { name = "a"; system = "foo"; builder = "/bin/sh"; __json = builtins.toJSON { a = 1; }; }' # Check for deprecation message -expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "In derivation 'a': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead." +expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead." # Check it works with the expected structured attrs hacky=$(nix-instantiate --expr "$hackyExpr") From 55eea4554203cc3486b80efe92a9261870edae34 Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Mon, 17 Nov 2025 18:55:06 -0500 Subject: [PATCH 357/364] Implement new builder-rpc-v0 derivation feature Expose a limited Nix daemon into derivations that request it so builders can add objects and register outputs before they complete. The manual is updated accordingly. --- doc/manual/source/store/building.md | 86 ++++++++++-- src/libexpr/include/nix/expr/eval.hh | 3 +- src/libexpr/primops.cc | 50 ++++++- .../build/derivation-building-goal.cc | 8 +- src/libstore/build/derivation-check.cc | 123 +++++++++++++----- src/libstore/build/derivation-check.hh | 16 ++- src/libstore/daemon.cc | 70 ++++++++-- .../nix/store/build/derivation-builder.hh | 9 +- src/libstore/include/nix/store/daemon.hh | 6 +- src/libstore/include/nix/store/derivations.hh | 5 + .../include/nix/store/remote-store.hh | 2 + .../include/nix/store/restricted-store.hh | 15 ++- src/libstore/include/nix/store/store-api.hh | 1 + .../include/nix/store/submit-store.hh | 6 + .../include/nix/store/worker-protocol.hh | 14 ++ src/libstore/remote-store.cc | 19 ++- src/libstore/restricted-store.cc | 7 + src/libstore/store-api.cc | 7 +- .../unix/build/derivation-builder-impl.hh | 41 ++++++ src/libstore/unix/build/derivation-builder.cc | 104 +++++++++++++-- src/libstore/worker-protocol.cc | 58 ++++++++- src/nix/meson.build | 1 + src/nix/store-submit-output.cc | 46 +++++++ src/nix/store-submit-output.md | 20 +++ src/nix/unix/daemon.cc | 6 +- .../dyn-drv/dep-built-drv-submitted.sh | 17 +++ tests/functional/dyn-drv/meson.build | 3 + .../dyn-drv/non-trivial-submitted.nix | 93 +++++++++++++ tests/functional/dyn-drv/submit-failure.nix | 45 +++++++ tests/functional/dyn-drv/submit-failure.sh | 17 +++ tests/functional/dyn-drv/submit-reference.nix | 45 +++++++ tests/functional/dyn-drv/submit-reference.sh | 37 ++++++ 32 files changed, 891 insertions(+), 89 deletions(-) create mode 100644 src/nix/store-submit-output.cc create mode 100644 src/nix/store-submit-output.md create mode 100644 tests/functional/dyn-drv/dep-built-drv-submitted.sh create mode 100644 tests/functional/dyn-drv/non-trivial-submitted.nix create mode 100644 tests/functional/dyn-drv/submit-failure.nix create mode 100755 tests/functional/dyn-drv/submit-failure.sh create mode 100644 tests/functional/dyn-drv/submit-reference.nix create mode 100755 tests/functional/dyn-drv/submit-reference.sh diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 087413406487..a97ffcd29cef 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -33,14 +33,14 @@ The life cycle of a build can be broken down into 3 parts: (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. In practice, Nix will store the log in `/nix/var/log/nix`) -3. Processing the outputs after the builder has exited. +3. Processing the outputs. - The builder process on exit should have left behind files for each output the derivation is supposed to produce. - The files must be processed to turn them into bona fide store objects. - If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + Traditionally, this happens only after the builder has exited: the builder process should have left behind files for each output the derivation is supposed to produce, and those files are processed to turn them into bona fide store objects. + Alternatively, the builder may send messages to Nix while it's running, creating filesystem objects and linking them to an output name. + This allows outputs to be processed concurrently during the build, allows outputs to depend on other newly created store objects, and also resolves some tricky issues with content-addressing and output-to-output references. + If the processing succeeds, the resulting store objects are associated with the derivation as (the results of) a successful build. -Step (3) is done by Nix externally to the build itself, which is just steps (1) and (2). -In step (3), just inert data is processed, since the builder process has exited or been killed by then. +Step (3) is done by Nix, either externally to the build (in the traditional case, operating on the inert data left behind after the builder has exited or been killed) or concurrently with it (in the IPC case). Step (1) however is best described not from Nix's perspective, but from the build process's perspective. > **Explanation** @@ -176,7 +176,11 @@ The builder is passed the arguments specified by the derivation attribute `args` ## Processing outputs -If the builder exited successfully, the following steps happen in order to turn the output directories left behind by the builder into proper store objects: +There are two methods for processing outputs. +But first, let us cover the requirements common to both methods. + +Regardless of which method is used, each output must be turned into a valid store object. +This involves two steps: - **Normalize the file permissions** @@ -189,15 +193,25 @@ If the builder exited successfully, the following steps happen in order to turn (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. - Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. +### Traditional (post-build) processing + +With the traditional method, the builder process on exit should have left behind files for each output the derivation is supposed to produce. +The files must be processed to turn them into bona fide store objects. +If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. +The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). +(This is not a special restriction for outputs; it is true for the references of all store objects in general.) - The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). - (This is not a special restriction for outputs; it is true for the references of all store objects in general.) +In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. +For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. +Scratch store paths must therefore be used instead. +Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. - In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. - For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. - Scratch store paths must therefore be used instead. - Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. +In addition to output-to-output references, rewriting is also needed to support self-references in the content-addressing case. +An output may contain its own store path digest, which is a self-reference. +Hash functions which are secure cannot allow the easy calculation of the quasi-fixed points needed to support self-references "natively", so instead we replace all would-be self-references with a sentinel value, and then rewrite the sentinel value to be the final store path digest. +Superficially, this post-hashing rewriting breaks the content address, but as the self-references are easily identified, the rewriting can be inverted to yield the original hashed data, allowing verifying the content address after all. At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. @@ -208,6 +222,50 @@ Additionally, those store objects (at least in the case that they are [content-a > The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. > The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. +### Concurrent processing via IPC + +With this method, the builder communicates with Nix during the build using inter-process communication (IPC). + +> **Implementation detail** +> +> The current implementation, `builder-rpc-v0`, exposes its interface over a limited form of the Nix daemon socket. +> Builders may use it either with their own implementation of the Nix protocol, or with the `nix store add` and `nix store submit-output` commands. +> +> Derivations with `builder-rpc-v0` in their set of [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures) +> will not receive output paths in their environment, and are expected to submit all outputs with the aforementioned commands or protocol. + +Instead of leaving files behind for Nix to process after exit, the builder explicitly requests the daemon create store objects one at a time, then sends commands assigning output names to the just-created store objects. + +Scanning for references proceeds as usual for each store object creation request, but the set of potential references to be scanned is greater: it includes both all inputs (as before) and also all previously-added store objects. +This means, if output `bar` is supposed to reference output `foo`, `foo` should be created first, and `bar` second. + +All store objects being created are content-addressed (there is no support for input-addressed outputs with the IPC approach). +When a store object is created, its content address store path will be calculated by Nix and then returned in the IPC response message. +The builder then knows what store path to use in subsequent store objects in order for reference scanning to pick them up. + +This overall approach has several advantages: + +- **No Nix-side rewriting** + + For content-addressed outputs, the builder is responsible for adding outputs in reference order, using the store paths from earlier adds in later ones. + This avoids the fragile rewriting that would otherwise be needed to fix up output-to-output references described above. + The builder, unlike Nix itself, is free to leverage domain-specific knowledge to do a better job. For example it can + + - uncompress, rewrite, and then recompress man pages, to not miss references hidden by compression. + + - make sure to rewrite data that is to be signed, like Apple binaries, before signing that data, so as not to invalidate any signatures by mistake. + +- **Pipelining** + + Downstream builds that only need some outputs (e.g., a "dev" or "headers" output) can start without waiting for all outputs to be ready. + Nix doesn't yet implement this, but it could and should. + +The major *disadvantage* of this approach is that it doesn't yet support self-references. +Unlike acyclic output-to-output references, self-references fundamentally do require rewriting. +The output-to-output case was only a challenge in the traditional case because all the outputs were submitted simultaneously, whereas the self-reference case is fundamentally challenging because of what it means for a hash function to be secure, as described above. +Neither batched (traditional) nor serial (IPC) submission of outputs can avoid this fundamental property of secure hash functions. +We could add support for such rewriting just for self-references, as is done for the traditional post-build processing, but we haven't yet done so as the very point of the IPC approach is to free Nix from any obligation to rewrite black-box data in unsound ways. + [references]: ./store-object.md#references [store path digest]: ./store-path.md#digest [store object]: ./store-object.md diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index f0a89343d856..86abb3c8f2ae 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -226,7 +226,7 @@ struct StaticEvalSymbols line, column, functor, toString, right, wrong, structuredAttrs, json, allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites, maxSize, maxClosureSize, builder, args, contentAddressed, impure, outputHash, outputHashAlgo, outputHashMode, recurseForDerivations, description, self, epsilon, startSet, - operator_, key, path, prefix, outputSpecified; + operator_, key, path, prefix, outputSpecified, requiredSystemFeatures; Expr::AstSymbols exprSymbols; @@ -279,6 +279,7 @@ struct StaticEvalSymbols .path = alloc.create("path"), .prefix = alloc.create("prefix"), .outputSpecified = alloc.create("outputSpecified"), + .requiredSystemFeatures = alloc.create("requiredSystemFeatures"), .exprSymbols = { .sub = alloc.create("__sub"), .lessThan = alloc.create("__lessThan"), diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 10023079f51c..8e01ce73639d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,8 +10,10 @@ #include "nix/store/names.hh" #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" +#include "nix/util/configuration.hh" #include "nix/util/mounted-source-accessor.hh" #include "nix/store/build.hh" +#include "nix/util/strings.hh" #include "nix/util/util.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" @@ -1523,6 +1525,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName bool contentAddressed = false; bool isImpure = false; + bool isSubmittingOutputs = false; std::optional outputHash; std::optional outputHashAlgo; std::optional ingestionMethod; @@ -1656,6 +1659,22 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName handleOutputs(ss); break; } + case EvalState::s.requiredSystemFeatures.getId(): { + /* Only parsed to detect `builder-rpc-v0`; skip + entirely unless the experimental feature is + enabled. */ + if (!experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations)) + break; + state.forceList(*i->value, pos, context_below); + for (auto elem : i->value->listView()) { + auto name = state.forceString(*elem, context, pos, context_below); + if (name == drvFeatureBuilderRpcV0) { + isSubmittingOutputs = true; + break; + } + } + break; + } default: break; } @@ -1677,6 +1696,15 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); + + /* Re-interpret the attribute's value as a list of + strings. + + We may wish to warn here better future-compat + later, e.g. requiring that it be a list of + strings without spaces to begin with. */ + auto forceStringList = [&] { return tokenizeString(s); }; + if (i->name == state.s.json) { warnAttr(HintFmt( "setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.")); @@ -1700,8 +1728,22 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName handleHashMode(s); break; case EvalState::s.outputs.getId(): - handleOutputs(tokenizeString(s)); + handleOutputs(forceStringList()); break; + case EvalState::s.requiredSystemFeatures.getId(): { + /* Only parsed to detect `builder-rpc-v0`; skip + entirely unless the experimental feature is + enabled. */ + if (!experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations)) + break; + for (auto & name : forceStringList()) { + if (name == drvFeatureBuilderRpcV0) { + isSubmittingOutputs = true; + break; + } + } + break; + } default: break; } @@ -1798,7 +1840,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName }, }; - drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + if (!isSubmittingOutputs) + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); drv.outputs.insert_or_assign("out", std::move(dof)); } @@ -1810,7 +1853,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { - drv.env[i] = hashPlaceholder(i); + if (!isSubmittingOutputs) + drv.env[i] = hashPlaceholder(i); if (isImpure) drv.outputs.insert_or_assign( i, diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 04f1c15b1714..37d9d2fc39fd 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -886,7 +886,11 @@ Goal::Co DerivationBuildingGoal::buildLocally( } void processDaemonConnection( - ref store, FdSource && from, FdSink && to, RestrictionContext & context) override + ref store, + FdSource && from, + FdSink && to, + RestrictionContext & context, + daemon::RecursiveFlag recursiveFlag) override { /** * TODO: We create a fresh Worker here because the @@ -898,7 +902,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( Worker freshWorker{goal.worker.store, goal.worker.evalStore}; auto builder = makeRestrictedBuilder(freshWorker, context); daemon::processConnection( - store, std::move(from), std::move(to), NotTrusted, daemon::Recursive, builder.get_ptr()); + store, std::move(from), std::move(to), NotTrusted, recursiveFlag, builder.get_ptr()); } }; diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index d0d9a8da76d3..d1562e811c7e 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -1,45 +1,88 @@ #include +#include "nix/store/derivations.hh" #include "nix/store/store-api.hh" #include "nix/store/build-result.hh" +#include "nix/util/hash.hh" #include "derivation-check.hh" namespace nix { -void checkCAFixedOutput( - StoreDirConfig & store, const StorePath & drvPath, const DerivationOutput & outputSpec, const ValidPathInfo & info) +void checkCAOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + const std::string & outputName) { - if (const auto * dof = std::get_if(&outputSpec.raw)) { - auto & wanted = dof->ca.hash; + std::visit( + overloaded{ + [&](const DerivationOutput::CAFixed & dof) { + auto & wanted = dof.ca.hash; - /* Check wanted hash */ - assert(info.ca); - auto & got = info.ca->hash; - if (wanted != got) { - throw BuildError( - BuildResult::Failure::HashMismatch, - "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true)); - } - if (!info.references.empty()) { - auto numViolations = info.references.size(); - throw BuildError( - BuildResult::Failure::HashMismatch, - "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*info.references.begin())); - } - } + /* Check wanted hash */ + assert(info.ca); + auto & got = info.ca->hash; + if (wanted != got) { + throw BuildError( + BuildResult::Failure::HashMismatch, + "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true)); + } + if (!info.references.empty()) { + auto numViolations = info.references.size(); + throw BuildError( + BuildResult::Failure::HashMismatch, + "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*info.references.begin())); + } + }, + [&](const DerivationOutput::CAFloating & dof) { + if (!info.ca.has_value()) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "floating content-addressing derivation '%s' output '%s' (at '%s') was not content-addressed", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path)); + } + if (info.ca->method != dof.method) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "content-addressing derivation '%s' output '%s' (at '%s') was hashed with method '%s', expected '%s'", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path), + info.ca->method.render(), + dof.method.render()); + } + if (info.ca->hash.algo != dof.hashAlgo) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "content-addressing derivation '%s' output '%s' (at '%s') was hashed with algorithm '%s', expected '%s'", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path), + printHashAlgo(info.ca->hash.algo), + printHashAlgo(dof.hashAlgo)); + } + }, + [&](const DerivationOutput::Deferred & _) {}, + [&](const DerivationOutput::Impure & _) {}, + [&](const DerivationOutput::InputAddressed & _) {}, + }, + outputSpec.raw); } void checkOutputs( Store & store, const StorePath & drvPath, - const decltype(Derivation::outputs) & drvOutputs, + const BasicDerivation & drv, const decltype(DerivationOptions::outputChecks) & outputChecks, const std::map & outputs) { @@ -53,10 +96,28 @@ void checkOutputs( const std::string & outputName = pair.first; const auto & info = pair.second; - auto * outputSpec = get(drvOutputs, outputName); - assert(outputSpec); + auto * outputSpec = get(drv.outputs, outputName); + if (!outputSpec) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "builder for '%s' submitted unknown output '%s' (Valid outputs are [%s])", + store.printStorePath(drvPath), + outputName, + concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; })); + } + + if (outputPathName(drv.name, outputName) != info.path.name()) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "derivation '%s' output '%s' (at '%s') was named '%s', expected '%s'", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path), + info.path.name(), + outputPathName(drv.name, outputName)); + } - checkCAFixedOutput(store, drvPath, *outputSpec, info); + checkCAOutput(store, drvPath, *outputSpec, info, outputName); /* Compute the closure and closure size of some output. This is slightly tricky because some of its references (namely @@ -122,8 +183,6 @@ void checkOutputs( if (auto output = get(outputs, refOutputName)) spec.insert(output->path); else { - std::string outputsListing = - concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); throw BuildError( BuildResult::Failure::OutputRejected, "derivation '%s' output check for '%s' contains output name '%s'," @@ -132,7 +191,7 @@ void checkOutputs( store.printStorePath(drvPath), outputName, refOutputName, - outputsListing); + concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; })); } }}, i); diff --git a/src/libstore/build/derivation-check.hh b/src/libstore/build/derivation-check.hh index f55f9aac2055..5c8c75da172d 100644 --- a/src/libstore/build/derivation-check.hh +++ b/src/libstore/build/derivation-check.hh @@ -8,12 +8,16 @@ namespace nix { /** - * If outputSpec is a CAFixed output, check that the actual output described in - * info meets the requirements for a CAFixed output. Do nothing if outputSpec is - * not a CAFixed output. + * If outputSpec is a CAFixed or CAFloating output, check that the actual output described in + * info meets the requirements for a CA output. + * Do nothing if outputSpec is not a CAFixed or CAFloating output. */ -void checkCAFixedOutput( - StoreDirConfig & store, const StorePath & drvPath, const DerivationOutput & outputSpec, const ValidPathInfo & info); +void checkCAOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + const std::string & outputName); /** * Check that outputs meets the requirements specified by the @@ -28,7 +32,7 @@ void checkCAFixedOutput( void checkOutputs( Store & store, const StorePath & drvPath, - const decltype(Derivation::outputs) & drvOutputs, + const BasicDerivation & drv, const decltype(DerivationOptions::outputChecks) & drvOptions, const std::map & outputs); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0c5673ea78d8..f8c5815a29af 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,4 +1,5 @@ #include "nix/store/daemon.hh" +#include "nix/util/configuration.hh" #include "nix/util/file-content-address.hh" #include "nix/util/signals.hh" #include "nix/store/worker-protocol.hh" @@ -13,7 +14,6 @@ #include "nix/store/indirect-root-store.hh" #include "nix/store/remote-store.hh" #include "nix/store/path-with-outputs.hh" -#include "nix/store/submit-store.hh" #include "nix/util/finally.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" @@ -315,6 +315,36 @@ static void performOp( WorkerProto::ReadConn rconn(conn); WorkerProto::WriteConn wconn(conn); + if (recursive == daemon::RecursiveFlag::RecursiveSubmitted) { + // Limit valid calls to reduce opportunities for nonreproducability in builds + // Since this is an allowlist, it's easiest to put it at the top before the switch + static constexpr std::array validOperations = { + // All the types of "Add" should be allowed + WorkerProto::Op::AddToStore, + WorkerProto::Op::AddMultipleToStore, + WorkerProto::Op::AddToStoreNar, + WorkerProto::Op::AddToStoreScanning, + // SubmitOutput is designed specifically for this use case + WorkerProto::Op::SubmitOutput, + // Used by nix cli, should never change actual outputs + WorkerProto::Op::AddTempRoot, + // Used by nix cli, restricted store will prevent it from seeing derivations it shouldn't + WorkerProto::Op::IsValidPath, + }; + if (std::ranges::find(validOperations, op) == validOperations.end()) { + throw Error("Operation %d not allowed inside derivation", op); + } + } else { + // Operations designed only for the experimental builder-rpc-v0 should never be exposed outside + // derivaitons that use it. + // AddToStoreScanning is still acceptable in ordinary recursive derivations, though. + // Throw the same error we do when using an unknown operation. + if (op == WorkerProto::Op::SubmitOutput + || (op == WorkerProto::Op::AddToStoreScanning && recursive == daemon::RecursiveFlag::NotRecursive)) { + throw Error("invalid operation %1%", op); + } + } + switch (op) { case WorkerProto::Op::IsValidPath: { @@ -803,7 +833,7 @@ static void performOp( // FIXME: use some setting in recursive mode. Will need to use // non-global variables. - if (!recursive) + if (recursive == RecursiveFlag::NotRecursive) clientSettings.apply(trusted); logger->stopWork(); @@ -1025,8 +1055,9 @@ static void performOp( if (!conn.protoVersion.features.contains(WorkerProto::featureAddToStoreScanning)) throw Error("Adding to store with scanning was requested, but not supported in negotiated protocol"); - if (!recursive) - throw Error("AddToStoreScanning only valid inside a `recursive-nix` derivation builder"); + if (recursive == daemon::RecursiveFlag::NotRecursive) + throw Error( + "AddToStoreScanning only valid within derivation with `builder-rpc-v0` or `recursive-nix` feature"); auto & submitStore = require(*store); @@ -1047,6 +1078,23 @@ static void performOp( break; } + case WorkerProto::Op::SubmitOutput: { + experimentalFeatureSettings.require(Xp::DynamicDerivations); + if (recursive != daemon::RecursiveFlag::RecursiveSubmitted) + throw Error("SubmitOutput only valid within derivation with `builder-rpc-v0` feature"); + + auto path = WorkerProto::Serialise::read(*store, rconn); + auto output = WorkerProto::Serialise::read(*store, rconn); + + auto & submitStore = require(*store); + + logger->startWork(); + submitStore.submitOutput(path, output); + logger->stopWork(); + conn.to << 1; + break; + } + default: throw Error("invalid operation %1%", op); } @@ -1061,7 +1109,7 @@ void processConnection( std::shared_ptr builder) { #ifndef _WIN32 // TODO need graceful async exit support on Windows? - auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; + auto monitor = (recursive == RecursiveFlag::NotRecursive) ? std::make_unique(from.fd) : nullptr; (void) monitor; // suppress warning ReceiveInterrupts receiveInterrupts; @@ -1080,10 +1128,16 @@ void processConnection( builder = store->getBuilder(); /* Exchange the greeting. */ - auto localVersion = WorkerProto::latest; - if (recursive) { + WorkerProto::Version localVersion; + + if (recursive == RecursiveFlag::RecursiveSubmitted) { + localVersion = WorkerProto::builderRpcV0; + } else if (recursive == RecursiveFlag::Recursive) { + localVersion = WorkerProto::latest; localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); localVersion.features.insert(std::string{WorkerProto::featureAddToStoreScanning}); + } else { + localVersion = WorkerProto::latest; } WorkerProto::BasicServerConnection conn; @@ -1098,7 +1152,7 @@ void processConnection( auto tunnelLogger = new TunnelLogger(conn.to, conn.protoVersion); auto prevLogger = logger; // FIXME - if (!recursive) { + if (recursive == RecursiveFlag::NotRecursive) { logger = tunnelLogger; applyJSONLogger(); } diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 08fce39b2777..088644a3eb32 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -5,6 +5,7 @@ #include #include "nix/store/build-result.hh" +#include "nix/store/daemon.hh" #include "nix/store/derivation-options.hh" #include "nix/store/build/derivation-building-misc.hh" #include "nix/store/derivations.hh" @@ -145,8 +146,12 @@ struct DerivationBuilderCallbacks * Process a recursive Nix daemon connection, using a builder * that enforces the restrictions of the given context. */ - virtual void - processDaemonConnection(ref store, FdSource && from, FdSink && to, RestrictionContext & context) = 0; + virtual void processDaemonConnection( + ref store, + FdSource && from, + FdSink && to, + RestrictionContext & context, + daemon::RecursiveFlag recursiveFlag) = 0; }; /** diff --git a/src/libstore/include/nix/store/daemon.hh b/src/libstore/include/nix/store/daemon.hh index a01205a976fd..a0a0d63e30b0 100644 --- a/src/libstore/include/nix/store/daemon.hh +++ b/src/libstore/include/nix/store/daemon.hh @@ -10,7 +10,11 @@ struct Builder; namespace daemon { -enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; +enum struct RecursiveFlag { + NotRecursive = 0, + Recursive = 1, + RecursiveSubmitted = 2, +}; void processConnection( ref store, diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 369b1ca30e2d..bfb20846ac34 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -16,6 +16,11 @@ namespace nix { +/** + * String to include in requiredSystemFeatures to enable builder-rpc-v0 + */ +static constexpr std::string_view drvFeatureBuilderRpcV0 = "builder-rpc-v0"; + struct StoreDirConfig; /* Abstract syntax of derivations. */ diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 295f37ab0313..1879786e2011 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -114,6 +114,8 @@ public: void registerDrvOutput(const Realisation & info) override; + void submitOutput(const SingleDerivedPath & path, const OutputName & output) override; + ref addToStoreScanning( Source & dump, std::string_view name, diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index 5b6e8b734c8e..92d7725a1fa3 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -64,6 +64,18 @@ public: virtual bool isAllowed(const DrvOutput & id) = 0; bool isAllowed(const DerivedPath & id); + /** + * Whether mounting dependencies inside the sandbox should happen, + * or if it should be entirely skipped. + */ + virtual bool shouldModifySandbox() = 0; + + /** + * Register a store path to an output name + * For builder-rpc-v0 + */ + virtual void submitOutput(const SingleDerivedPath & path, const OutputName & output) = 0; + /** * Add 'path' to the set of paths that may be referenced by the * outputs, and make it appear in the sandbox. @@ -92,7 +104,8 @@ public: } try { - addDependencyImpl(path); + if (shouldModifySandbox()) + addDependencyImpl(path); promise.set_value(); } catch (...) { /* Notify all other waiters that we are done. */ diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index cd512f4c0f74..404944b834a2 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/store/outputs-spec.hh" #include "nix/store/path.hh" #include "nix/store/derived-path.hh" #include "nix/util/hash.hh" diff --git a/src/libstore/include/nix/store/submit-store.hh b/src/libstore/include/nix/store/submit-store.hh index 48bea62faa8c..df3d60878561 100644 --- a/src/libstore/include/nix/store/submit-store.hh +++ b/src/libstore/include/nix/store/submit-store.hh @@ -13,6 +13,12 @@ private: public: inline static std::string operationName = "Submit outputs for a currently running derivation"; + /** + * Submit an output for the current derivation. + * Only makes sense when running within a recursive-nix derivation + */ + virtual void submitOutput(const SingleDerivedPath & path, const OutputName & output) = 0; + /** * Add to store, scanning references. * Only within a recursive-nix derivation, as there would otherwise be no known diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 5e924c28c45f..67c7395d599f 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -119,6 +119,12 @@ struct WorkerProto static const Version minimum; + /** + * Static version for the `builder-rpc-v0` feature. + * Should never change, as any modification would be derivation-visible. + */ + static const Version builderRpcV0; + /** * Feature for transmitting `UnkeyedRealisation` and `DrvOutput` * using drvPath (store path) instead of the old hash-based JSON format. @@ -140,6 +146,11 @@ struct WorkerProto */ static constexpr std::string_view featureAddToStoreScanning = "add-to-store-scanning"; + /** + * Feature for enabling the `SubmitOutput` operation + */ + static constexpr std::string_view featureSubmitOutput = "submit-output"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -264,6 +275,7 @@ enum struct WorkerProto::Op : uint64_t { // QueryActiveBuilds = 48, // reserved for https://github.com/NixOS/nix/pull/15979 // AddTempRoots = 49, // reserved for https://github.com/NixOS/nix/pull/16113 // QueryPathInfos = 50, // reserved for https://github.com/DeterminateSystems/nix-src/pull/539 + SubmitOutput = 1000, // Only used within derivations with feature AddToStoreScanning = 1001, }; @@ -334,6 +346,8 @@ inline std::ostream & operator<<(std::ostream & s, WorkerProto::Op op) template<> DECLARE_WORKER_SERIALISER(DerivedPath); template<> +DECLARE_WORKER_SERIALISER(SingleDerivedPath); +template<> DECLARE_WORKER_SERIALISER(BuildResult); template<> DECLARE_WORKER_SERIALISER(KeyedBuildResult); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b96a4326a32b..9a5dfddfae61 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -91,12 +91,13 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - // The DisableSetOptions and `AddToStoreScanning` features aren't in the `latest` constant because it is - // shared with the daemon, which only adds the feature under certain conditions. + // The following features aren't in the `latest` constant because it is + // shared with the daemon, which only adds the features under certain conditions. // Adding is easier than removing. auto localVersion = WorkerProto::latest; localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); localVersion.features.insert(std::string{WorkerProto::featureAddToStoreScanning}); + localVersion.features.insert(std::string{WorkerProto::featureSubmitOutput}); conn.protoVersion = WorkerProto::BasicClientConnection::handshake(conn.to, tee, localVersion); if (conn.protoVersion.number < WorkerProto::minimum.number) @@ -506,6 +507,20 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn.processStderr(); } +void RemoteStore::submitOutput(const SingleDerivedPath & path, const OutputName & output) +{ + auto conn(getConnection()); + if (!conn->protoVersion.features.contains(WorkerProto::featureSubmitOutput)) + throw Error( + "the daemon does not support SubmitOutput, perhaps this is not in a derivation with the `builder-rpc-v0` feature?"); + + conn->to << WorkerProto::Op::SubmitOutput; + WorkerProto::Serialise::write(*this, *conn, path); + conn->to << output; + conn.processStderr(); + readInt(conn->from); +} + ref RemoteStore::addToStoreScanning( Source & dump, std::string_view name, diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 3fd9187e6219..c1b2853a05e9 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -115,6 +115,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void registerDrvOutput(const Realisation & info) override; + void submitOutput(const SingleDerivedPath & path, const OutputName & output) override; + ref addToStoreScanning( Source & dump, std::string_view name, @@ -284,6 +286,11 @@ void RestrictedStore::registerDrvOutput(const Realisation & info) throw Error("registerDrvOutput"); } +void RestrictedStore::submitOutput(const SingleDerivedPath & path, const OutputName & output) +{ + goal.submitOutput(path, output); +} + ref RestrictedStore::addToStoreScanning( Source & dump, std::string_view name, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e1d7493f7b67..133d84087abc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -396,8 +396,13 @@ StringSet Store::Config::getDefaultSystemFeatures() if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); - if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) + if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) { res.insert("recursive-nix"); + } + + if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations)) { + res.insert(std::string{drvFeatureBuilderRpcV0}); + } return res; } diff --git a/src/libstore/unix/build/derivation-builder-impl.hh b/src/libstore/unix/build/derivation-builder-impl.hh index cdfe9a901791..b21587692f9c 100644 --- a/src/libstore/unix/build/derivation-builder-impl.hh +++ b/src/libstore/unix/build/derivation-builder-impl.hh @@ -129,6 +129,15 @@ protected: */ OutputPathMap scratchOutputs; + /** + * Whether or not derivation is using outputs submitted via recursive-nix + */ + bool usingSubmitted; + /** + * Output paths from the `SubmitOutput` store command + */ + Sync submittedOutputs; + const static std::filesystem::path homeDir; /** @@ -175,6 +184,33 @@ protected: bool isAllowed(const DerivedPath & req); + bool shouldModifySandbox() override + { + return !usingSubmitted; + } + + void submitOutput(const SingleDerivedPath & path, const OutputName & output) override + { + auto submittedOutputs(this->submittedOutputs.lock()); + + auto * opaque = std::get_if(&path.raw()); + if (!opaque) + throw Error( + "Attempted to submit Built path '%s' for output '%s'.\n" + " Only Opaque paths are supported, see https://github.com/NixOS/nix/issues/12727", + path.to_string(store), + output); + + if (submittedOutputs->contains(output)) + throw Error( + "Attempted to submit duplicate output '%s' (old '%s', new '%s')", + output, + store.printStorePath(*get(*submittedOutputs, output)), + store.printStorePath(opaque->path)); + + submittedOutputs->insert_or_assign(output, opaque->path); + }; + friend struct RestrictedStore; /** @@ -366,6 +402,11 @@ private: */ SingleDrvOutputs registerOutputs(); + /** + * Check that the derivation outputs submitted by recursive-nix exist + * and attach them to the derivation + */ + SingleDrvOutputs checkSubmittedOutputs(); protected: /** diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 24c2437fd57f..40df622fda01 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1,4 +1,5 @@ #include "nix/store/build/derivation-builder.hh" +#include "nix/util/configuration.hh" #include "nix/util/file-system-at.hh" #include "nix/util/file-system.hh" #include "nix/store/local-store.hh" @@ -210,7 +211,7 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild() root. */ killSandbox(true); - /* Terminate the recursive Nix daemon. */ + /* Terminate the recursive Nix daemons. */ stopDaemon(); if (buildResult.cpuUser && buildResult.cpuSystem) { @@ -238,9 +239,14 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild() }; } - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = registerOutputs(); + SingleDrvOutputs builtOutputs; + if (usingSubmitted) { + builtOutputs = checkSubmittedOutputs(); + } else { + /* Compute the FS closure of the outputs and register them as + being valid. */ + builtOutputs = registerOutputs(); + } cleanupBuild(true); @@ -468,7 +474,15 @@ std::optional DerivationBuilderImpl::startBuild() /* Fire up a Nix daemon to process recursive Nix calls from the builder. */ - if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix")) + auto requiredFeatures = drvOptions.getRequiredSystemFeatures(drv); + + usingSubmitted = requiredFeatures.count(drvFeatureBuilderRpcV0); + + if (usingSubmitted && !drv.type().isCA()) { + throw Error("The builder-rpc-v0 feature may only be used with content-addressing derivations"); + } + + if (usingSubmitted || requiredFeatures.count("recursive-nix")) startDaemon(); /* Run the builder. */ @@ -783,7 +797,11 @@ void DerivationBuilderImpl::initEnv() void DerivationBuilderImpl::startDaemon() { - experimentalFeatureSettings.require(Xp::RecursiveNix); + if (usingSubmitted) { + experimentalFeatureSettings.require(Xp::DynamicDerivations); + } else { + experimentalFeatureSettings.require(Xp::RecursiveNix); + } auto store = makeRestrictedStore( [&] { @@ -806,7 +824,14 @@ void DerivationBuilderImpl::startDaemon() chownToBuilder(socketPath); - daemonThread = std::thread([this, store]() { + daemon::RecursiveFlag recursiveFlag; + if (usingSubmitted) { + recursiveFlag = daemon::RecursiveFlag::RecursiveSubmitted; + } else { + recursiveFlag = daemon::RecursiveFlag::Recursive; + } + + daemonThread = std::thread([this, store, recursiveFlag]() { while (true) { /* Accept a connection. */ @@ -828,9 +853,10 @@ void DerivationBuilderImpl::startDaemon() auto doneFlag = make_ref(); - auto workerThread = std::thread([this, doneFlag, store, remote{std::move(remote)}]() { + auto workerThread = std::thread([this, doneFlag, store, remote{std::move(remote)}, recursiveFlag]() { try { - miscMethods->processDaemonConnection(store, FdSource(remote.get()), FdSink(remote.get()), *this); + miscMethods->processDaemonConnection( + store, FdSource(remote.get()), FdSink(remote.get()), *this, recursiveFlag); debug("terminated daemon connection"); } catch (const Interrupted &) { debug("interrupted daemon connection"); @@ -1515,7 +1541,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() PathFmt(store.toRealPath(newInfo.path))); deletePath(actualPath); /* Trigger the hash-mismatch error. */ - checkCAFixedOutput(store, drvPath, *output, newInfo); + checkCAOutput(store, drvPath, *output, newInfo, outputName); unreachable(); } } @@ -1626,7 +1652,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() /* Apply output checks. This includes checking of the wanted vs got hash of fixed-outputs. */ - checkOutputs(store, drvPath, drv.outputs, drvOptions.outputChecks, infos); + checkOutputs(store, drvPath, drv, drvOptions.outputChecks, infos); if (buildMode == bmCheck) { return {}; @@ -1672,6 +1698,62 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() return builtOutputs; } +SingleDrvOutputs DerivationBuilderImpl::checkSubmittedOutputs() +{ + // Submitted outputs from the recursive nix daemon + // It's fine to lock here since all other threads with the reference have been shut down. + auto submittedOutputs(this->submittedOutputs.lock()); + + SingleDrvOutputs builtOutputs; + + std::map infos; + + for (auto & [outputName, outputPath] : *submittedOutputs) { + infos.emplace(outputName, *store.queryPathInfo(outputPath)); + } + + // checkOutputs only performs checks that make sense for both submitting and non-submitting derivations, + // more verification steps needed afterward + checkOutputs(store, drvPath, drv, drvOptions.outputChecks, infos); + + for (auto & [outputName, output] : drv.outputs) { + // For some reason cannot be moved to checkOutputs, needs debugging + if (!submittedOutputs->contains(outputName)) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "builder for '%s' failed to submit output path for '%s'", + store.printStorePath(drvPath), + outputName); + } + + // We should have already checked that the derivation is content-addressing in startBuild + // and that the outputs of a content-addressing derivation is content-addressed in checkOutputs. + // Add an assert here just in case, but it should never trigger. + assert( + std::get_if(&output.raw) + || std::get_if(&output.raw)); + + // No need to sign CA outputs, only the realisation matters + auto realisation = Realisation{ + { + .outPath = *get(*submittedOutputs, outputName), + }, + DrvOutput{ + .drvPath = drvPath, + .outputName = outputName, + }, + }; + + store.signRealisation(realisation); + store.registerDrvOutput(realisation); + builtOutputs.emplace(outputName, realisation); + + // TODO: handle --check + } + + return builtOutputs; +} + void DerivationBuilderImpl::cleanupBuild(bool force) { if (force) { diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 37aa4e31bec2..5eaae231a50d 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -25,9 +25,7 @@ const WorkerProto::Version WorkerProto::latest = { }, .features = { - std::string{ - WorkerProto::featureRealisationWithPath, - }, + std::string{WorkerProto::featureRealisationWithPath}, std::string{WorkerProto::featureDeleteDeadSpecificReferrers}, }, }; @@ -40,6 +38,21 @@ const WorkerProto::Version WorkerProto::minimum = { }, }; +const WorkerProto::Version WorkerProto::builderRpcV0 = { + .number = + { + .major = 1, + .minor = 38, + }, + .features = + { + std::string{WorkerProto::featureRealisationWithPath}, + std::string{WorkerProto::featureDisableSetOptions}, + std::string{WorkerProto::featureAddToStoreScanning}, + std::string{WorkerProto::featureSubmitOutput}, + }, +}; + std::partial_ordering WorkerProto::Version::operator<=>(const WorkerProto::Version & other) const { auto numCmp = number <=> other.number; @@ -227,6 +240,45 @@ void WorkerProto::Serialise::write( } } +SingleDerivedPath +WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto tag = readNum(conn.from); + switch (tag) { + case 0: + return SingleDerivedPath::Opaque{ + .path = WorkerProto::Serialise::read(store, conn), + }; + case 1: { + auto drvPath = make_ref(WorkerProto::Serialise::read(store, conn)); + return SingleDerivedPath::Built{ + .drvPath = std::move(drvPath), + .output = readString(conn.from), + }; + } + default: + throw Error("Invalid tag %d for single derived path", tag); + } +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WorkerProto::WriteConn conn, const SingleDerivedPath & req) +{ + std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & o) { + conn.to << uint8_t{0}; + WorkerProto::write(store, conn, o.path); + }, + [&](const SingleDerivedPath::Built & b) { + conn.to << uint8_t{1}; + WorkerProto::write(store, conn, *b.drvPath); + conn.to << b.output; + }, + }, + req.raw()); +} + KeyedBuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { diff --git a/src/nix/meson.build b/src/nix/meson.build index 45d584335702..b4a28c7ab7e5 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -120,6 +120,7 @@ nix_sources = [ config_priv_h ] + files( 'store-gc.cc', 'store-info.cc', 'store-repair.cc', + 'store-submit-output.cc', 'store.cc', 'upgrade-nix.cc', 'verify.cc', diff --git a/src/nix/store-submit-output.cc b/src/nix/store-submit-output.cc new file mode 100644 index 000000000000..c23c5d688da7 --- /dev/null +++ b/src/nix/store-submit-output.cc @@ -0,0 +1,46 @@ +#include "nix/cmd/command.hh" +#include "nix/store/store-api.hh" +#include "nix/store/store-cast.hh" +#include "nix/store/submit-store.hh" + +namespace nix { + +struct CmdSubmitOutput : StoreCommand +{ + std::string path; + OutputName output; + + CmdSubmitOutput() + { + expectArg("path", &path); + expectArg("output", &output); + } + + std::string description() override + { + return "submit a store object as one of the outputs of the derivation currently being built"; + } + + std::string doc() override + { + return +#include "store-submit-output.md" + ; + } + + std::optional experimentalFeature() override + { + return Xp::DynamicDerivations; + } + + void run(ref store) override + { + auto & submitStore = require(*store); + auto path = SingleDerivedPath::parse(*store, this->path); + submitStore.submitOutput(path, output); + } +}; + +static auto rCmdSubmitOutput = registerCommand2({"store", "submit-output"}); + +} // namespace nix diff --git a/src/nix/store-submit-output.md b/src/nix/store-submit-output.md new file mode 100644 index 000000000000..17b73686d257 --- /dev/null +++ b/src/nix/store-submit-output.md @@ -0,0 +1,20 @@ +R""( + +# Examples + +* To submit a given [store object] as the output named `out`: + + ```console + # nix store submit-output /nix/store/h6zs50y2662apmnbcnhnbxll76lv02yy-hello-2.12.3 out + ``` + +# Description + +`nix store submit-output` registers a [store object] as an output of the currently-running derivation. + +It only functions when running inside a content-addressing derivation with the `builder-rpc-v0` +system feature, which provides a limited daemon socket to the builder. +Execution in any other environment will fail. + +[store object]: @docroot@/store/store-object.md +)"" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 956adc43c99b..527840b2c025 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -375,7 +375,8 @@ static void daemonLoop( // Handle the connection. auto store = storeConfig->openStore(); store->init(); - processConnection(store, FdSource(remote.get()), FdSink(remote.get()), trusted, NotRecursive); + processConnection( + store, FdSource(remote.get()), FdSink(remote.get()), trusted, RecursiveFlag::NotRecursive); exit(0); }, @@ -437,7 +438,8 @@ static void forwardStdioConnection(RemoteStore & store) */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - processConnection(store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, daemon::NotRecursive); + processConnection( + store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, daemon::RecursiveFlag::NotRecursive); } /** diff --git a/tests/functional/dyn-drv/dep-built-drv-submitted.sh b/tests/functional/dyn-drv/dep-built-drv-submitted.sh new file mode 100644 index 000000000000..41add90e0404 --- /dev/null +++ b/tests/functional/dyn-drv/dep-built-drv-submitted.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source common.sh + +# builder-rpc-v0 +requireDaemonNewerThan "2.35pre20260507" + +TODO_NixOS # can't enable a sandbox feature easily + +enableFeatures 'ca-derivations' +restartDaemon + +NIX_BIN_DIR="$(dirname "$(type -p nix)")" +export NIX_BIN_DIR + +nix build -L --file ./non-trivial-submitted.nix unstructured --no-link +nix build -L --file ./non-trivial-submitted.nix structured --no-link diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build index 9936a8b5c0cd..22dda3bcb0e9 100644 --- a/tests/functional/dyn-drv/meson.build +++ b/tests/functional/dyn-drv/meson.build @@ -10,6 +10,9 @@ suites += { 'dep-built-drv.sh', 'old-daemon-error-hack.sh', 'dep-built-drv-2.sh', + 'dep-built-drv-submitted.sh', + 'submit-failure.sh', + 'submit-reference.sh', ], 'workdir' : meson.current_source_dir(), } diff --git a/tests/functional/dyn-drv/non-trivial-submitted.nix b/tests/functional/dyn-drv/non-trivial-submitted.nix new file mode 100644 index 000000000000..c26c45c7e107 --- /dev/null +++ b/tests/functional/dyn-drv/non-trivial-submitted.nix @@ -0,0 +1,93 @@ +with import ./config.nix; + +let + baseAttrs = { + name = "build-e.drv"; + + requiredSystemFeatures = [ "builder-rpc-v0" ]; + + buildCommand = '' + set -e + set -u + + if [[ ! -z "''${out+set}" ]]; then + echo "out variable set in builder-rpc-v0 derivation" + exit 1 + fi + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + export NIX_CONFIG='extra-experimental-features = nix-command ca-derivations dynamic-derivations' + + declare -A deps=( + [a]="" + [b]="a" + [c]="a" + [d]="b c" + [e]="b c d" + ) + + # Cannot just literally include this, or Nix will think it is the + # *outer* derivation that's trying to refer to itself, and + # substitute the string too soon. + placeholder=$(nix eval --raw --expr 'builtins.placeholder "out"') + + declare -A drvs=() + for word in a b c d e; do + inputDrvs="" + for dep in ''${deps[$word]}; do + if [[ "$inputDrvs" != "" ]]; then + inputDrvs+="," + fi + read -r -d "" line <> \"\$out\""], + "builder": "${shell}", + "env": { + "out": "$placeholder", + "$word": "hello, from $word!", + "PATH": ${builtins.toJSON path} + }, + "inputs": { + "drvs": { + $inputDrvs + }, + "srcs": [] + }, + "name": "build-$word", + "outputs": { + "out": { + "method": "nar", + "hashAlgo": "sha256" + } + }, + "system": "${system}", + "version": 4 + } + EOF + drvPath=$(echo "$json" | nix derivation add) + storeDir=$(dirname "$drvPath") + drvs[$word]="$(basename "$drvPath")" + done + nix store submit-output "''${storeDir}/''${drvs[e]}" out + ''; + + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + }; + + buildDynamic = attrs: builtins.outputOf (mkDerivation attrs).outPath "out"; +in +{ + unstructured = buildDynamic baseAttrs; + structured = buildDynamic (baseAttrs // { __structuredAttrs = true; }); +} diff --git a/tests/functional/dyn-drv/submit-failure.nix b/tests/functional/dyn-drv/submit-failure.nix new file mode 100644 index 000000000000..d0c4e8566016 --- /dev/null +++ b/tests/functional/dyn-drv/submit-failure.nix @@ -0,0 +1,45 @@ +with import ./config.nix; + +let + buildSubmitting = + name: command: + mkDerivation { + inherit name; + + requiredSystemFeatures = [ "builder-rpc-v0" ]; + + buildCommand = '' + set -e + set -u + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + export NIX_CONFIG='extra-experimental-features = nix-command ca-derivations dynamic-derivations' + + ${command} + ''; + + __contentAddressed = true; + outputHashMode = "nar"; + outputHashAlgo = "sha256"; + + }; +in +{ + duplicate = buildSubmitting "duplicate" '' + mkdir a + echo "miao" > a/gatto + a="$(nix store add -n duplicate ./a)" + nix store submit-output "$a" out + + mkdir b + echo "miau" > b/katze + b="$(nix store add -n duplicate ./b)" + nix store submit-output "$b" out + ''; + noSubmit = buildSubmitting "no-submit" '' + mkdir a + echo "nyaa" > a/neko + a="$(nix store add -n no-submit ./a)" + # Don't run the required `nix store submit-output` + ''; +} diff --git a/tests/functional/dyn-drv/submit-failure.sh b/tests/functional/dyn-drv/submit-failure.sh new file mode 100755 index 000000000000..5b370be155ae --- /dev/null +++ b/tests/functional/dyn-drv/submit-failure.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source common.sh + +# builder-rpc-v0 +requireDaemonNewerThan "2.35pre20260507" + +TODO_NixOS + +enableFeatures 'dynamic-derivations ca-derivations' +restartDaemon + +NIX_BIN_DIR="$(dirname "$(type -p nix)")" +export NIX_BIN_DIR + +expectStderr 1 nix build -L --file ./submit-failure.nix noSubmit --no-link | grepQuiet "failed to submit output" +expectStderr 1 nix build -L --file ./submit-failure.nix duplicate --no-link | grepQuiet "submit duplicate output" diff --git a/tests/functional/dyn-drv/submit-reference.nix b/tests/functional/dyn-drv/submit-reference.nix new file mode 100644 index 000000000000..dd4d4ab2d2f5 --- /dev/null +++ b/tests/functional/dyn-drv/submit-reference.nix @@ -0,0 +1,45 @@ +with import ./config.nix; + +let + buildSubmitting = + name: command: + mkDerivation { + inherit name; + + requiredSystemFeatures = [ "builder-rpc-v0" ]; + + buildCommand = '' + set -e + set -u + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + export NIX_CONFIG='extra-experimental-features = nix-command ca-derivations dynamic-derivations' + + ${command} + ''; + + __contentAddressed = true; + outputHashMode = "nar"; + outputHashAlgo = "sha256"; + }; + + dependency = buildSubmitting "dependency" '' + mkdir dependency + echo "this is a dependency" > dependency/foo + out="$(nix store add --scan ./dependency)" + nix store submit-output "$out" out + ''; +in +buildSubmitting "reference" '' + mkdir mao + echo "miao" > mao/foo + echo "${dependency}" > mao/reference + mao="$(nix store add --scan ./mao)" + + mkdir felis + echo "miau" > felis/foo + echo "$mao" > felis/reference + felis="$(nix store add --scan -n reference ./felis)" + + nix store submit-output "$felis" out +'' diff --git a/tests/functional/dyn-drv/submit-reference.sh b/tests/functional/dyn-drv/submit-reference.sh new file mode 100755 index 000000000000..d1774f2d44ca --- /dev/null +++ b/tests/functional/dyn-drv/submit-reference.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +source common.sh + +listReferences() { + nix path-info "$1" --json --json-format 2 | \ + jq -r '.info[].references | sort | .[]' +} + +# builder-rpc-v0 +requireDaemonNewerThan "2.35pre20260507" + +TODO_NixOS + +enableFeatures 'dynamic-derivations ca-derivations' +restartDaemon + +NIX_BIN_DIR="$(dirname "$(type -p nix)")" +export NIX_BIN_DIR + +outPath="$(nix build -L --file ./submit-reference.nix --no-link --print-out-paths)" + +mapfile -t rootRefs < <(listReferences "${outPath}") +if [[ ${#rootRefs[@]} -ne 1 ]]; then + echo "Incorrect references for root output" >&2 + exit 1 +fi + +echo "${rootRefs[0]}" | grep -- "-mao$" + +mapfile -t secondRefs < <(listReferences "$NIX_STORE_DIR/${rootRefs[0]}") +if [[ ${#secondRefs[@]} -ne 1 ]]; then + echo "Incorrect references for other output" >&2 + exit 1 +fi + +echo "${secondRefs[0]}" | grep -- "-dependency$" From 6d9834295b3162c04b0c2c94b6369562b5b89b00 Mon Sep 17 00:00:00 2001 From: figsoda Date: Wed, 22 Jul 2026 11:00:25 -0400 Subject: [PATCH 358/364] flake: drop unused gitignore override warning: input 'git-hooks-nix' has an override for a non-existent input 'gitignore' removed upstream in https://github.com/cachix/git-hooks.nix/pull/721 --- flake.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/flake.nix b/flake.nix index ac2bd551121c..62944b794957 100644 --- a/flake.nix +++ b/flake.nix @@ -18,7 +18,6 @@ inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 inputs.git-hooks-nix.inputs.flake-compat.follows = ""; - inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ From 30820a54b112f4842bdb7df28b61b2a607e54033 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 18:33:52 +0200 Subject: [PATCH 359/364] Populate the srcToStore cache when we have a fetcher cache hit This saves tens of thousands of calls to SQLite and to makeFixedOutputPathFromCA(). (E.g. instantiating nixpkgs#firefox was doing 4744 fetcher cache checks for /pkgs/stdenv/generic/source-stdenv.sh.) --- src/libfetchers/fetch-to-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index acedca6dbed1..085c97da7d01 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -82,6 +82,7 @@ std::pair fetchToStore2( path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode)); return {storePath, hash}; } debug("source path '%s' not in store", path); From 181f52a8762538ee22d1e31c20908e1711107e54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 22:00:05 +0200 Subject: [PATCH 360/364] Fix assertion failure in processSandboxSetupMessages() This fixes the following Sentry crash report: libnixutil.so.2.34.80xff807ce5e404 nix::panic (error.cc:459) libnixutil.so.2.34.80xff807ceed68c __wrap___assert_fail (wrap-assert-fail.cc:18) libnixutil.so.2.34.80xff807cee2834 nix::Pid::wait (processes.cc:100) libnixstore.so.2.34.80xff807cc4fcfc operator() (derivation-builder.cc:1102) libnixstore.so.2.34.80xff807cc4fcfc nix::DerivationBuilderImpl::processSandboxSetupMessages (derivation-builder.cc:1111) libnixstore.so.2.34.80xff807cc5d390 .LTHUNK39.lto_priv.4 (linux-derivation-builder.cc:477) libnixstore.so.2.34.80xff807cc4b10c nix::DerivationBuilderImpl::startBuild (derivation-builder.cc:895) This happens because in the Linux sandbox, if the sandbox helper fails, `pid` is not initialised yet so we can't wait for it. --- src/libstore/unix/build/derivation-builder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 40df622fda01..d3e8133c0bfe 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -679,12 +679,12 @@ void DerivationBuilderImpl::processSandboxSetupMessages() try { return readLine(builderOut.get()); } catch (Error & e) { - auto status = pid.wait(); + auto status = pid != -1 ? pid.wait() : 0; e.addTrace( {}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", store.printStorePath(drvPath), - statusToString(status), + status ? statusToString(status) : "no status", concatStringsSep("|", msgs)); throw; } From 88579a87f43a88b8ea5c0a0bbaa737620ad1241e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Jul 2026 23:10:11 +0300 Subject: [PATCH 361/364] libutil-tests: Add test for sourceToSink boost.context bug This surfaces the bug fixed by https://github.com/boostorg/context/pull/337 in our test suite. --- src/libutil-tests/serialise.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 01e5579117e1..24ef6d2c60f0 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -1,5 +1,6 @@ #include "nix/util/serialise.hh" +#include #include namespace nix { @@ -37,4 +38,34 @@ TEST(readPadding, works) } } +TEST(sourceToSink, forcedUnwindUcaughtExceptions) +{ + int uncaughtExceptions = 42; + bool caught = false; + + auto sink = sourceToSink([&](Source & source) { + auto recordUncaughtExceptions = Finally([&]() { uncaughtExceptions = std::uncaught_exceptions(); }); + try { + StringSink s; + source.drainInto(s, 8); + source.drainInto(s, 8); + } catch (const boost::context::detail::forced_unwind &) { + caught = true; + throw; + } + }); + + *sink << 42; + + // Abandon the coroutine. This will trigger it to unwind with boost::context::detail::forced_unwind. + sink.reset(); + + ASSERT_TRUE(caught); + // This is a test for boost.context regression fixed by https://github.com/boostorg/context/pull/337. + // Without the fix std::uncaught_exceptions() *misreports* 0 while there's stack unwinding in progress. + // The issue only surfaces with libstdc++ and fcontext when boost.context + // uses fiber-specific exception states and messes with __cxa_get_globals(). + ASSERT_EQ(uncaughtExceptions, 0); +} + } // namespace nix From f8b102f902ee98bf051232a001e1e55b13ff70a4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Jul 2026 23:01:17 +0300 Subject: [PATCH 362/364] Add boost.context patch for std::uncaught_exceptions() misreporting 0 while abandoning the coroutine Applies a patch to our boost.context dependency to fix https://github.com/NixOS/nix/issues/16174. An alternative would be to apply basically the same workaround to our suspension points, but that would be more fragile. Other packagers might want to apply the boost patch too (maybe once that's merged upstream), since the issue is likely to affect more stuff than just nix. The bug is subtle enough that it went unnoticed for quite some time. --- packaging/dependencies.nix | 3 + ...eptions-not-accounting-for-forced_un.patch | 102 ++++++++++++++++++ src/libutil-tests/serialise.cc | 2 +- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index db7948566f72..f8f64abf3968 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -126,6 +126,9 @@ scope: { "--with-iostreams" "--with-url" ]; + patches = [ + ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch + ]; enableIcu = false; }).overrideAttrs (old: { diff --git a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch new file mode 100644 index 000000000000..7ec13724c40b --- /dev/null +++ b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch @@ -0,0 +1,102 @@ +From 5883212311535a0046031d74d1568ae173c1e35b Mon Sep 17 00:00:00 2001 +From: Sergei Zimmerman +Date: Tue, 21 Jul 2026 21:15:51 +0000 +Subject: [PATCH] Fix uncaught_exceptions() not accounting for forced_unwind + +Unwound fibers would see std::uncaught_exceptions() == 0, while a +forced_unwind exception is in "flight". This goes against the contract +of std::uncaught_exceptions() that scope guards rely upon. Failing +to report the correct number of uncaught exceptions (especially +misreporting zero) will lead to scope guards to misbehave badly and skip +running cleanup code which branches on whether the destructor is called +during stack unwinding or not. + +This is because the "throw" would happen before the destructor is run on +the fiber stack being switched to, but the increment would be clobbered +by the destructor of manage_exception_state. + +I'm not sure what the contract of run ontop_fcontext is wrt to whether +the the caller provided function can throw or not, but in my best +understanding the forced_unwind mechanism is mostly internal and so is +throwing from ontop_fcontext in the switched-to fiber. Thus, I've kept +the catch block scoped to detail::forced_unwind. +--- + include/boost/context/fiber_fcontext.hpp | 37 +++++++++++++++++------- + test/test_fiber.cpp | 24 +++++++++++++++ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/include/boost/context/fiber_fcontext.hpp b/include/boost/context/fiber_fcontext.hpp +index 543ba6c..38476c9 100644 +--- a/boost/context/fiber_fcontext.hpp ++++ b/boost/context/fiber_fcontext.hpp +@@ -70,7 +70,9 @@ namespace context { + namespace detail { + + // manage_exception_state is a dummy struct unless we have specific support +-struct manage_exception_state {}; ++struct manage_exception_state { ++ void from_forced_unwind() noexcept {} ++}; + + } // namespace detail + } // namespace context +@@ -90,6 +92,11 @@ public: + manage_exception_state() { + exception_state_ = *__cxa_get_globals(); + } ++ // Hack to account for the forced_unwind exception thrown in fiber_unwind ++ // that's run ontop before the destructor. ++ void from_forced_unwind() noexcept { ++ exception_state_.uncaughtExceptions += 1; ++ } + ~manage_exception_state() { + *__cxa_get_globals() = exception_state_; + } +@@ -376,13 +383,18 @@ public: + BOOST_ASSERT( nullptr != fctx_); + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); +- return { detail::jump_fcontext( ++ try { ++ return { detail::jump_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- nullptr).fctx }; ++ nullptr).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + template< typename Fn > +@@ -391,14 +403,19 @@ public: + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); + auto p = std::forward< Fn >( fn); +- return { detail::ontop_fcontext( ++ try { ++ return { detail::ontop_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- & p, +- detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ & p, ++ detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + explicit operator bool() const noexcept { diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 24ef6d2c60f0..8af591a5e6e2 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -65,7 +65,7 @@ TEST(sourceToSink, forcedUnwindUcaughtExceptions) // Without the fix std::uncaught_exceptions() *misreports* 0 while there's stack unwinding in progress. // The issue only surfaces with libstdc++ and fcontext when boost.context // uses fiber-specific exception states and messes with __cxa_get_globals(). - ASSERT_EQ(uncaughtExceptions, 0); + ASSERT_EQ(uncaughtExceptions, 1); } } // namespace nix From 54aa8880be35e7d169e8d84d3dc135893cc4ff12 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Jul 2026 00:04:26 +0300 Subject: [PATCH 363/364] libutil-tests: Disable sourceToSink.forcedUnwindUcaughtExceptions under ASan --- src/libutil-tests/serialise.cc | 10 ++++++++++ src/libutil/meson.build | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 8af591a5e6e2..40424c6853a6 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -1,4 +1,5 @@ #include "nix/util/serialise.hh" +#include "nix/util/config.hh" #include #include @@ -38,6 +39,13 @@ TEST(readPadding, works) } } +// The following test catches the bug only under fcontext backend, +// but Boost.Coroutine2 stack switching when abandoning confuses the +// hell out of ASan. The workaround to getting ASan working isn't +// immediately useful because it works only with ucontext implementation. +// https://www.boost.org/doc/libs/1_89_0/libs/coroutine2/doc/html/coroutine2/stack/sanitizers.html +#if !NIX_ASAN_ENABLED + TEST(sourceToSink, forcedUnwindUcaughtExceptions) { int uncaughtExceptions = 42; @@ -68,4 +76,6 @@ TEST(sourceToSink, forcedUnwindUcaughtExceptions) ASSERT_EQ(uncaughtExceptions, 1); } +#endif + } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index fad7167617a3..b61403b51c4f 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -44,6 +44,12 @@ configdata_pub.set( description : 'Whether nix has been built with UBSan enabled', ) +configdata_pub.set( + 'NIX_ASAN_ENABLED', + ('address' in get_option('b_sanitize')).to_int(), + description : 'Whether nix has been built with ASan enabled', +) + subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' From 7de2962f62e4688f22b286292d8772da86d4dc3f Mon Sep 17 00:00:00 2001 From: ArkhamKnight25 Date: Sun, 21 Jun 2026 18:18:25 +0900 Subject: [PATCH 364/364] libstore: ship build inputs with the build request Add buildDerivation() and buildPathsWithResults() overloads to the Builder interface that take a StorePathSet of inputs and copy them into the builder's store before building, instead of relying on a separate copyPaths() beforehand. Shipping the inputs together with the build request lets a builder store decide where to build based on those inputs (for example, choosing a machine by total input size) before they are copied. This is not possible when copying and building are two separate calls. RemoteBuilder and LegacySSHBuilder open the local store as the source and copy the inputs into the builder's store; RestrictedBuilder rejects the operation. The copy honours the builders-use-substitutes setting. build-remote folds its standalone dependency copy into these overloads: the trusted/CA path calls buildDerivation() with the input paths, and the untrusted path ships the inputs together with the derivation's closure via buildPathsWithResults(). --- src/libstore/build/entry-points.cc | 32 +++++++++++++++++++ src/libstore/include/nix/store/build.hh | 32 +++++++++++++++++++ .../include/nix/store/build/worker.hh | 14 ++++++++ src/libstore/legacy-ssh-store.cc | 22 +++++++++++++ src/libstore/remote-store.cc | 28 ++++++++++++++++ src/libstore/restricted-store.cc | 21 ++++++++++++ src/nix/build-remote/build-remote.cc | 26 +++++++-------- 7 files changed, 161 insertions(+), 14 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index fdbe5f269821..bf919e443e6b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,7 +1,9 @@ #include "nix/store/derivations.hh" #include "nix/store/build/worker.hh" +#include "nix/store/worker-settings.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" +#include "nix/store/store-open.hh" #include "nix/util/strings.hh" #include @@ -23,6 +25,18 @@ BuildResult LocalBuilder::buildDerivation(const StorePath & drvPath, const Basic return getWorker()->buildDerivation(drvPath, drv, buildMode); } +BuildResult LocalBuilder::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, const StorePathSet & inputs, BuildMode buildMode) +{ + return getWorker()->buildDerivation(drvPath, drv, inputs, buildMode); +} + +std::vector LocalBuilder::buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) +{ + return getWorker()->buildPathsWithResults(reqs, inputs, buildMode); +} + void LocalBuilder::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ @@ -121,6 +135,24 @@ BuildResult Worker::buildDerivation(const StorePath & drvPath, const BasicDeriva }; } +BuildResult Worker::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, const StorePathSet & inputs, BuildMode buildMode) +{ + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, store, inputs, NoRepair, NoCheckSigs, substitute); + return buildDerivation(drvPath, drv, buildMode); +} + +std::vector +Worker::buildPathsWithResults(const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) +{ + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, store, inputs, NoRepair, NoCheckSigs, substitute); + return buildPathsWithResults(reqs, buildMode); +} + void Worker::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ diff --git a/src/libstore/include/nix/store/build.hh b/src/libstore/include/nix/store/build.hh index dbf2fc76ce7b..2c7bb1211804 100644 --- a/src/libstore/include/nix/store/build.hh +++ b/src/libstore/include/nix/store/build.hh @@ -78,6 +78,38 @@ struct Builder virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; + /** + * Like the other buildDerivation(), but additionally copies a set of + * input paths into the builder's store before the build is run. + * + * This lets the caller ship the build inputs together with the build + * request, rather than as a separate prior `copyPaths()`. Whether the + * inputs are fetched via substitution or copied directly is governed by + * the `builders-use-substitutes` setting. + * + * @param inputs The store paths to make available in the builder's + * store before building. For a remote builder these are copied across + * the connection; for the local `Worker` they are copied from the eval + * store. + */ + virtual BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode = bmNormal) = 0; + + /** + * Like the other buildPathsWithResults(), but additionally copies a set + * of input paths into the builder's store before building. + * + * @param inputs The store paths to make available in the builder's + * store before building, copied subject to the + * `builders-use-substitutes` setting (see the buildDerivation() overload + * above). + */ + virtual std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode = bmNormal) = 0; + /** * Ensure that a path is valid. If it is not currently valid, it * may be made valid by running a substitute (if defined for the diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 9af1997e34dd..bf1726f32b90 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -85,6 +85,13 @@ public: std::vector buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override; + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override; void ensurePath(const StorePath & path) override; void repairPath(const StorePath & path) override; @@ -431,6 +438,13 @@ public: std::vector buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override; + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override; void ensurePath(const StorePath & path) override; void repairPath(const StorePath & path) override; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1765c5af747a..5b46732cbda3 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -9,6 +9,7 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/store/build-result.hh" #include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/ssh.hh" #include "nix/store/derivations.hh" @@ -47,6 +48,27 @@ struct LegacySSHBuilder : Builder BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override + { + auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, *store, inputs, NoRepair, NoCheckSigs, substitute); + return buildDerivation(drvPath, drv, buildMode); + } + + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override + { + auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, *store, inputs, NoRepair, NoCheckSigs, substitute); + return buildPathsWithResults(reqs, buildMode); + } + /** * Note, the returned function must only be called once, or we'll * try to read from the connection twice. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9a5dfddfae61..e94ff774a63a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -2,6 +2,7 @@ #include "nix/store/path.hh" #include "nix/store/store-api.hh" #include "nix/util/file-content-address.hh" +#include "nix/store/store-open.hh" #include "nix/util/serialise.hh" #include "nix/util/util.hh" #include "nix/store/path-with-outputs.hh" @@ -599,6 +600,15 @@ struct RemoteBuilder : Builder BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override; + + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override; + void ensurePath(const StorePath & path) override; /** @@ -724,6 +734,24 @@ BuildResult RemoteBuilder::buildDerivation(const StorePath & drvPath, const Basi return WorkerProto::Serialise::read(*store, *conn); } +BuildResult RemoteBuilder::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, const StorePathSet & inputs, BuildMode buildMode) +{ + auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, *store, inputs, NoRepair, NoCheckSigs, substitute); + return buildDerivation(drvPath, drv, buildMode); +} + +std::vector RemoteBuilder::buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) +{ + auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, *store, inputs, NoRepair, NoCheckSigs, substitute); + return buildPathsWithResults(reqs, buildMode); +} + void RemoteBuilder::ensurePath(const StorePath & path) { auto conn(store->getConnection()); diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index c1b2853a05e9..53ab790d9499 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -192,6 +192,15 @@ struct RestrictedBuilder : Builder BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override; + + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override; + void ensurePath(const StorePath & path) override; void repairPath(const StorePath & path) override; @@ -404,6 +413,18 @@ RestrictedBuilder::buildDerivation(const StorePath & drvPath, const BasicDerivat throw Unsupported("buildDerivation"); } +BuildResult RestrictedBuilder::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, const StorePathSet & inputs, BuildMode buildMode) +{ + throw Unsupported("buildDerivation (with inputs)"); +} + +std::vector RestrictedBuilder::buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) +{ + throw Unsupported("buildPathsWithResults (with inputs)"); +} + void RestrictedBuilder::repairPath(const StorePath & path) { throw Unsupported("repairPath"); diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 19d4ab095e61..8c52d2556973 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -301,15 +301,10 @@ static int main_build_remote(int argc, char ** argv) signal(SIGALRM, old); } - auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute; - - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); - } - uploadLock = -1; + auto inputPaths = store->parseStorePathSet(inputs); + auto drv = store->readDerivation(*drvPath); std::optional optResult; @@ -339,7 +334,7 @@ static int main_build_remote(int argc, char ** argv) // // 2. Changing the `inputSrcs` set changes the // associated output ids, which break CA derivations - .inputs = drv.inputs.drvs.map.empty() ? drv.inputs.srcs : store->parseStorePathSet(inputs), + .inputs = drv.inputs.drvs.map.empty() ? drv.inputs.srcs : inputPaths, .platform = drv.platform, .builder = drv.builder, .args = drv.args, @@ -347,7 +342,7 @@ static int main_build_remote(int argc, char ** argv) .structuredAttrs = drv.structuredAttrs, .name = drv.name, }; - optResult = sshStore->getBuilder()->buildDerivation(*drvPath, resolvedDrv); + optResult = sshStore->getBuilder()->buildDerivation(*drvPath, resolvedDrv, inputPaths); auto & result = *optResult; if (auto * failureP = result.tryGetFailure()) { if (settings.keepFailed) { @@ -361,11 +356,14 @@ static int main_build_remote(int argc, char ** argv) "build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, failureP->message()); } } else { - copyClosure(*store, *sshStore, StorePathSet{*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->getBuilder()->buildPathsWithResults({DerivedPath::Built{ - .drvPath = makeConstantStorePathRef(*drvPath), - .outputs = OutputsSpec::All{}, - }}); + auto inputPathsWithDrv = inputPaths; + store->computeFSClosure(*drvPath, inputPathsWithDrv); + auto res = sshStore->getBuilder()->buildPathsWithResults( + {DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All{}, + }}, + inputPathsWithDrv); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]);