diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index af94c3e9e5bb..d802a82c2ecd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -44,10 +44,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open bug issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index fe9f9dd209d4..2238b88386ae 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -29,10 +29,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open feature issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index 070e0bd9b25b..965d4db6fd96 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -37,10 +37,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open installer issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 4e05b626d398..5675bf2391a6 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -21,10 +21,12 @@ assignees: '' - [ ] checked [latest Nix manual] \([source]) - [ ] checked [open documentation issues and pull requests] for possible duplicates +- [ ] reviewed the [contributing guide] [latest Nix manual]: https://nix.dev/manual/nix/development/ [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation +[contributing guide]: https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md --- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c155bf8bfa4f..e861dcfc5fa0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,7 @@ so you understand the process and the expectations. - what information to include in commit messages - proper attribution - volunteering contributions effectively +- AI/automation policy - how to get help and our review process. PR stuck in review? We have two Nix team meetings per week online that are open for everyone in a jitsi conference: diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index 535ae9d08fd9..2fb433b741c3 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -4,22 +4,12 @@ inputs: dogfood: description: "Whether to use Nix installed from the latest artifact from master branch" required: true # Be explicit about the fact that we are using unreleased artifacts - experimental-installer: - description: "Whether to use the experimental installer to install Nix" - default: false - experimental-installer-version: - description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used." - # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one - default: "latest" extra_nix_config: description: "Gets appended to `/etc/nix/nix.conf` if passed." install_url: description: "URL of the Nix installer" required: false - default: "https://releases.nixos.org/nix/nix-2.32.1/install" - tarball_url: - description: "URL of the Nix tarball to use with the experimental installer" - required: false + default: "https://releases.nixos.org/nix/nix-2.34.8/install" github_token: description: "Github token" required: true @@ -51,74 +41,14 @@ runs: gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" env: GH_TOKEN: ${{ inputs.github_token }} DOGFOOD_REPO: "NixOS/nix" - - name: "Gather system info for experimental installer" - shell: bash - if: ${{ inputs.experimental-installer == 'true' }} - run: | - echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)" - - if [ "$RUNNER_OS" == "Linux" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="linux" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - elif [ "$RUNNER_OS" == "macOS" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="darwin" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - if [ "$RUNNER_ARCH" == "X64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=x86_64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - elif [ "$RUNNER_ARCH" == "ARM64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=aarch64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH" - exit 1 - fi - - echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - env: - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - name: "Download latest experimental installer" - shell: bash - id: download-latest-experimental-installer - if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }} - run: | - RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId") - - EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT" - mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - # Executable permissions are lost in artifacts - find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} + - echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ inputs.github_token }} - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 - if: ${{ inputs.experimental-installer != 'true' }} with: # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} extra_nix_config: ${{ inputs.extra_nix_config }} - - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20 - if: ${{ inputs.experimental-installer == 'true' }} - with: - diagnostic-endpoint: "" - # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/`? - local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }} - source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} - nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} - extra-conf: ${{ inputs.extra_nix_config }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a1f2..1880a89da556 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "nix" + directory: "/" + schedule: + interval: "weekly" + groups: + flake-inputs: + patterns: + - "*" diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 2d01cfefc295..9bc2ee17e672 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -20,13 +20,13 @@ jobs: with: app-id: ${{ vars.CI_APP_ID }} private-key: ${{ secrets.CI_APP_PRIVATE_KEY }} - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches fetch-depth: 0 - name: Create backport PRs - uses: korthout/backport-action@3c06f323a58619da1e8522229ebc8d5de2633e46 # v4.3.0 + uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8 # v4.6.0 id: backport with: # Config README: https://github.com/korthout/backport-action#backport-action diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be8cb0f0c4fd..4e603cb1c085 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: eval: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -40,7 +40,7 @@ jobs: name: pre-commit checks runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: ./.github/actions/install-nix-action with: dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} @@ -87,7 +87,7 @@ jobs: runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -144,7 +144,7 @@ jobs: continue-on-error: true timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -164,23 +164,23 @@ jobs: - scenario: on ubuntu runs-on: ubuntu-24.04 os: linux - experimental-installer: false + rust-installer: false - scenario: on macos runs-on: macos-14 os: darwin - experimental-installer: false - - scenario: on ubuntu (experimental) + rust-installer: false + - scenario: on ubuntu (rust) runs-on: ubuntu-24.04 os: linux - experimental-installer: true - - scenario: on macos (experimental) + rust-installer: true + - scenario: on macos (rust) runs-on: macos-14 os: darwin - experimental-installer: true + rust-installer: true name: installer test ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Download installer tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -188,22 +188,19 @@ jobs: path: out - name: Looking up the installer tarball URL id: installer-tarball-url - run: | - echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 - if: ${{ !matrix.experimental-installer }} + run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 + if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - uses: ./.github/actions/install-nix-action - if: ${{ matrix.experimental-installer }} - with: - dogfood: false - experimental-installer: true - tarball_url: ${{ steps.installer-tarball-url.outputs.tarball-path }} - github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Run rust installer + if: ${{ matrix.rust-installer }} + run: | + chmod +x out/nix-installer + ./out/nix-installer install --no-confirm + env: + RUST_BACKTRACE: full - run: sudo apt install fish zsh if: matrix.os == 'linux' - run: brew install fish @@ -220,7 +217,7 @@ jobs: runs-on: ubuntu-24.04 name: clang-tidy steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action @@ -238,14 +235,14 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout nix - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Checkout flake-regressions - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: NixOS/flake-regressions path: flake-regressions - name: Checkout flake-regressions-data - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: NixOS/flake-regressions-data path: flake-regressions/tests @@ -258,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} @@ -275,7 +272,7 @@ jobs: github.event_name == 'push' && github.ref_name == 'master' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: ./.github/actions/install-nix-action diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index cd21336c8913..332115cfd992 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: eval_id: - description: "Hydra evaluation ID" + description: "Hydra evaluation ID (from the maintenance-X.Y-release jobset)" required: true type: number is_latest: @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-24.04 environment: releases steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/install-nix-action with: dogfood: false # Use stable version @@ -34,7 +34,7 @@ jobs: # get the same uberhack that nix-shell has to support it. echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV" - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 with: role-to-assume: "arn:aws:iam::080433136561:role/nix-release" role-session-name: nix-release-oidc-${{ github.run_id }} @@ -51,12 +51,12 @@ jobs: echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null sudo systemctl restart docker - name: Login to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.gitignore b/.gitignore index 7fd5574cae92..c0d09b7e189c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,23 +5,6 @@ src/.wraplock # Python bytecode (clang-tidy runner scripts) __pycache__/ -# /tests/functional/ -/tests/functional/common/subst-vars.sh -/tests/functional/restricted-innocent -/tests/functional/debugger-test-out -/tests/functional/test-libstoreconsumer/test-libstoreconsumer -/tests/functional/nix-shell - -# /tests/functional/lang/ -/tests/functional/lang/*.out -/tests/functional/lang/*.out.xml -/tests/functional/lang/*.err -/tests/functional/lang/*.ast - -# /tests/functional/cli-characterisation/ -/tests/functional/cli-characterisation/*.out -/tests/functional/cli-characterisation/*.err - /outputs *~ diff --git a/.version b/.version index aa5388f63762..3a05135cd86d 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.0 +2.36.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c170ae4a770..f6679e0a8c93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. + * Review the **Automation/AI Policy** below. 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. @@ -87,6 +88,66 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). +## Automation/AI policy + +Every contribution to Nix and related development venues, including code, documentation, and communication on GitHub and Matrix, must have a responsible person in the loop who is accountable for that contribution and reviews it before submission, and must transparently disclose any non‐trivial use of automation to produce it, including but not limited to LLM‐based AI tools. + +Human communication must remain human. Pull request / issue descriptions and comments, documentation, commit messages, and code comments must be human-authored. + +The following sections give more detail. + +### Scope + +Any use of automated tools to generate non‐trivial amounts of output as part of a contribution, in whole or in part, verbatim or edited, is covered by this policy, except as listed in the Exemptions section. +Both LLM‐based AI tools and hand‐written automation are covered. +Contributions include code and documentation in commits, commit messages, pull request summaries and reviews, issue and vulnerability reports, GitHub comments, Matrix messages, and Discourse posts. +The covered venues are the GitHub repositories for Nix and related projects under the jurisdiction of the Nix team, Matrix rooms that are focused on development of those projects, and Discourse topics about Nix development. + +PRs that seek to address issues that appear easier to fix, such as those marked with [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue), are held to the same standard as other issues. +Just because the problem seems "easy" and more likely to be successfully fixed by an unsupervised agent does not mean bending the rules is permissible. +Even the most trivial changes still must have a responsible person in the loop. + +### Accountability + +Everyone who submits a contribution to Nix is responsible for it, regardless of the use of automated tooling. +Before submission, they must establish a reasonable level of understanding of the contribution and expectation of its correctness. +A contributor submitting a contribution intended for inclusion in Nix is also responsible for ensuring that it is [appropriately licensed](https://github.com/NixOS/nix/blob/master/COPYING) and credited, and not encumbered by any incompatible copyright. + +When output from automated tooling is used in contributions, a contributor must establish confidence in that output. + +This policy applies equally to any further discussion of a contribution. +Comments and reviews must separately satisfy the same requirements of understanding, review, and disclosure. +Contributors are expected to be able to answer questions about their contribution and respond to feedback appropriately, **without simply forwarding messages back and forth to automated tools**. + +It is not permitted to submit automated contributions without any manual review or intervention, outside of standard community automation. +Automation without any manual review must not be used as the sole arbiter of whether to merge a change. + +### Transparency + +All covered use of automated tooling for a contribution must be disclosed as part of that contribution. + +In the case of LLM‐based AI tooling used for commits, this **must** be in the form of an `Assisted-by:` Git commit trailer, including at least the tool name and the primary model name and version used for the contribution. When using unreleased models, it is acceptable to say "unspecified". +A `Co-authored-by:` trailer does not satisfy this policy. + +Any adequate form of disclosure is permitted for other kinds of tooling and contribution. + +### Exemptions + +The following situations are fully or partially exempt: + +* Use of standard deterministic editor/IDE/formatter/text transformation tooling to produce changes that the author manually reviews and understands is exempt, including inline "auto‐completion" (even if LLM‐based) of short, rote snippets of text that do not contribute anything beyond boilerplate the author would have written anyway, and spelling and grammar checkers. + +* Use of standard community automation is exempt (e.g. dependabot). + +* Use of AI tools for research, testing, debugging, or review is out of scope, if no substantial amount of their output is included in the resulting contribution. + However, if these tools had a significant technical influence on your contribution, you are still responsible for it per the Accountability section, and are expected to disclose this where relevant. + +* Use of machine translation for commit and pull request descriptions and comments is exempt from the requirement to understand the translated output. + However, the requirements of appropriate confidence in the original text, responsibility, and disclosure still apply, and you should additionally include the original untranslated contribution. + +* Use of automation in a contribution clearly marked as not being ready for merge (e.g. a draft pull request) is exempt from the requirement for full self‐review, as long as some amount of review has been done and it is expected that the requirements will be met by the time it is marked as ready. + This does not waive any other requirement. + ## Making changes to the Nix manual The Nix reference manual is hosted on https://nix.dev/manual/nix. diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 5e1b23c7a36b..8d2383a92fb9 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -57,8 +57,6 @@ rec { nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); diff --git a/ci/gha/tests/prepare-installer-for-github-actions b/ci/gha/tests/prepare-installer-for-github-actions index 0fbecf25c2aa..e240e56fca0a 100755 --- a/ci/gha/tests/prepare-installer-for-github-actions +++ b/ci/gha/tests/prepare-installer-for-github-actions @@ -2,10 +2,14 @@ set -euo pipefail -nix build -L ".#installerScriptForGHA" ".#binaryTarball" +nix build -L \ + ".#installerScriptForGHA" \ + ".#binaryTarball" \ + ".#rustInstaller" mkdir -p out cp ./result/install "out/install" name="$(basename "$(realpath ./result-1)")" # everything before the first dash cp -r ./result-1 "out/${name%%-*}" +cp ./result-2/bin/nix-installer "out/nix-installer" diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e66611affe08..a58d4f79e132 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -46,10 +46,12 @@ let ## Settings - ${showSettings { - prefix = "store-${slug}"; - inherit inlineHTML; - } settings} + ${replaceStrings [ "@store-slug@" ] [ "store-${slug}" ] ( + showSettings { + prefix = "store-${slug}"; + inherit inlineHTML; + } settings + )} ''; experimentalFeatureNote = optionalString (experimentalFeature != null) '' diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fd841e80cbb..f8c37b75b301 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -1,7 +1,7 @@ project( 'nix-manual', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -241,7 +241,9 @@ foreach command : nix_nested_manpages meson.current_source_dir() / 'source', meson.current_build_dir() / 'source', doc_url, - meson.current_source_dir() / 'source/command-ref' / command[0] / (page + '.md'), + meson.current_source_dir() / 'source/command-ref' / command[0] / ( + page + '.md' + ), '@OUTPUT0@', ], input : [ @@ -257,6 +259,7 @@ foreach command : nix_nested_manpages endforeach nix3_manpages = [ + 'nix', 'nix3-build', 'nix3-bundle', 'nix3-config', @@ -264,29 +267,34 @@ nix3_manpages = [ 'nix3-config-show', 'nix3-copy', 'nix3-daemon', - 'nix3-derivation-add', 'nix3-derivation', + 'nix3-derivation-add', 'nix3-derivation-show', 'nix3-develop', 'nix3-edit', + 'nix3-env', 'nix3-env-shell', 'nix3-eval', + 'nix3-flake', 'nix3-flake-archive', 'nix3-flake-check', 'nix3-flake-clone', 'nix3-flake-info', 'nix3-flake-init', 'nix3-flake-lock', - 'nix3-flake', 'nix3-flake-metadata', 'nix3-flake-new', 'nix3-flake-prefetch', + 'nix3-flake-prefetch-inputs', 'nix3-flake-show', 'nix3-flake-update', 'nix3-fmt', - 'nix3-hash-file', + 'nix3-formatter', + 'nix3-formatter-build', + 'nix3-formatter-run', 'nix3-hash', 'nix3-hash-convert', + 'nix3-hash-file', 'nix3-hash-path', 'nix3-hash-to-base16', 'nix3-hash-to-base32', @@ -294,15 +302,15 @@ nix3_manpages = [ 'nix3-hash-to-sri', 'nix3-help', 'nix3-help-stores', + 'nix3-key', 'nix3-key-convert-secret-to-public', 'nix3-key-generate-secret', - 'nix3-key', 'nix3-log', + 'nix3-nar', 'nix3-nar-cat', 'nix3-nar-dump-path', 'nix3-nar-ls', 'nix3-nar-pack', - 'nix3-nar', 'nix3-path-info', 'nix3-print-dev-env', 'nix3-profile', @@ -314,19 +322,21 @@ nix3_manpages = [ 'nix3-profile-rollback', 'nix3-profile-upgrade', 'nix3-profile-wipe-history', - 'nix3-realisation-info', - 'nix3-realisation', + 'nix3-registry', 'nix3-registry-add', 'nix3-registry-list', - 'nix3-registry', 'nix3-registry-pin', 'nix3-registry-remove', + 'nix3-registry-resolve', 'nix3-repl', 'nix3-run', 'nix3-search', + 'nix3-store', 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', + 'nix3-store-build-trace-info', + 'nix3-store-build-trace', 'nix3-store-cat', 'nix3-store-copy-log', 'nix3-store-copy-sigs', @@ -337,16 +347,15 @@ nix3_manpages = [ 'nix3-store-info', 'nix3-store-ls', 'nix3-store-make-content-addressed', - 'nix3-store', 'nix3-store-optimise', 'nix3-store-path-from-hash-part', 'nix3-store-prefetch-file', 'nix3-store-repair', + 'nix3-store-roots-daemon', 'nix3-store-sign', 'nix3-store-verify', 'nix3-upgrade-nix', 'nix3-why-depends', - 'nix', ] foreach page : nix3_manpages diff --git a/doc/manual/package.nix b/doc/manual/package.nix index af5e6cf1c229..817b05dbc1c0 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -144,6 +144,21 @@ mkMesonDerivation (finalAttrs: { # Exclude undocumented builtins ".*/language/builtins\\.html#builtins-addErrorContext" ".*/language/builtins\\.html#builtins-appendContext" + # `print.html` aggregates content from all pages, including + # the JSON schema pages and builtins pages excluded above, + # so it inherits the same broken fragment links. + ".*/print\\.html#algorithm" + ".*/print\\.html#root" + ".*/print\\.html#builtins-addErrorContext" + ".*/print\\.html#builtins-appendContext" + ".*/print\\.html#derivations_pattern1_structuredAttrs_additionalProperties" + ".*/print\\.html#structuredAttrs_additionalProperties" + ]; + # `404.html` uses `` so that absolute links + # work on the deployed site. Lychee cannot resolve `/` against + # a local file path, so skip the file entirely. + exclude_path = [ + ".*/404\\.html" ]; }; }; diff --git a/doc/manual/redirects.json b/doc/manual/redirects.json index 0a6c71508006..04e14aacc3d5 100644 --- a/doc/manual/redirects.json +++ b/doc/manual/redirects.json @@ -337,7 +337,7 @@ "string-literal": "string-literals.html" }, "language/derivations.html": { - "builder-execution": "../store/building.html#builder-execution" + "builder-execution": "../store/building.html" }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", @@ -353,7 +353,6 @@ "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", "integration-tests": "testing.html#integration-tests", "installer-tests": "testing.html#installer-tests", - "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", "characterization-testing": "testing.html#characterisation-testing-unit", @@ -363,6 +362,9 @@ "reverting": "contributing.html#reverting", "branches": "contributing.html#branches" }, + "store/store-path.html": { + "store-directory": "#store-directory-path" + }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", "package-attribute-set": "#package", diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py index 6da4c19b0ce7..a0fcb6a55776 100644 --- a/doc/manual/remove_before_wrapper.py +++ b/doc/manual/remove_before_wrapper.py @@ -22,7 +22,7 @@ def main(): shutil.rmtree(output, ignore_errors=True) shutil.rmtree(output_temp, ignore_errors=True) - # Execute nix command with `--write-to` tempary output + # Execute nix command with `--write-to` temporary output nix_command_write_to = nix_command + ['--write-to', output_temp] subprocess.run(nix_command_write_to, check=True) diff --git a/doc/manual/rl-next/async-post-build-hook.md b/doc/manual/rl-next/async-post-build-hook.md deleted file mode 100644 index ea061f14ae9d..000000000000 --- a/doc/manual/rl-next/async-post-build-hook.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: Make post-build-hook asynchronous -prs: [15451] -issues: [15406] ---- - -This change makes the `post-build-hook` run asynchronously but still as part of the goal. -This retains the current behavior that a waiting goal will not start until the `post-build-hook` of the goal it is waiting on completes. -However, multiple `post-build-hook`s can now run concurrently just as multiple goals can run concurrently. diff --git a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md deleted file mode 100644 index 5034c65438cb..000000000000 --- a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -synopsis: S3 substituters fall back to the URL's region for STS WebIdentity auth -prs: [15594] ---- - -When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, -GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL -as a fallback for the STS endpoint region if neither `AWS_REGION` nor -`AWS_DEFAULT_REGION` is set. Previously, IRSA setups that exported -`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail -with a misleading "IMDS provider" error. diff --git a/doc/manual/rl-next/beta-installer b/doc/manual/rl-next/beta-installer deleted file mode 100644 index b02564d95d2d..000000000000 --- a/doc/manual/rl-next/beta-installer +++ /dev/null @@ -1,29 +0,0 @@ ---- -synopsis: "Rust nix-installer in beta" -prs: [] ---- - -The Rust-based rewrite of the Nix installer is now in beta. -We'd love help testing it out! - -To test out the new installer, run: -``` -curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install -``` - -This installer can be run even when you have an existing, script-based Nix installation without any adjustments. - -This new installer also comes with the ability to uninstall your Nix installation; run: -``` -/nix/nix-installer uninstall -``` - -This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation). - -This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems. -Thanks to Determinate Systems for all the investment they've put into the installer. - -Source for the installer is in https://github.com/NixOS/nix-installer. -Report any issues in that repo. - -For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action. diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md deleted file mode 100644 index 61b6bc51b1c0..000000000000 --- a/doc/manual/rl-next/build-trace-rework.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -synopsis: "Content-addressed derivations: realisations keyed by store path instead of hash modulo" -issues: [11897] -prs: [12464] ---- - -The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. This affects the **binary cache protocol** and the **wire protocols**. - -### What changed - -#### Build trace format - -Previously, a build trace entry (realisation) was keyed by the **hash modulo** of the derivation. -A SHA-256 hash computed via the complex "derivation hash modulo" algorithm. -This required implementations to understand ATerm serialisation and the full derivation hashing scheme just to look up or store build results. - -Now, build trace entries are keyed by the **regular derivation store path** plus the output name. For example, instead of: - -``` -sha256:ba7816bf8f01...!out -``` - -The key is now: - -``` -/nix/store/abc...-foo.drv^out -``` - -This is simpler, more intuitive, and means that third-party tools implementing CA derivation support (e.g., Hydra) -no longer need to implement the derivation hash modulo algorithm. - -#### Build trace usage - -Previously the build trace contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. -Now, it only contains entries for resolved derivations. -For now, unresolved derivations will be resolved from these underlying build trace entries. -This is slower, but avoids a bunch of correctness issues. - -### Binary cache protocol - -- The directory for build traces moved from `realisations/` to `build-trace-v2/`. -- File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. -- The JSON format of build trace entries is now split into `key` and `value` objects: - ```json - { - "key": { - "drvPath": "abc...-foo.drv", - "outputName": "out" - }, - "value": { - "outPath": "xyz...-foo", - "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] - } - } - ``` - Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. -- The deprecated `dependentRealisations` field has been removed. - -Existing binary caches will need to be re-populated with the new format for CA derivation build traces. -Old build traces at the previous URLs are simply abandoned. -Non-CA builds are unaffected. - -### Wire protocols - -- **Worker protocol**: - A new feature flag `realisation-with-path-not-hash` is negotiated during the handshake. - Clients and daemons that both support this feature use the new binary serialisation for `DrvOutput`, `UnkeyedRealisation`, and related types. - Fallback to older protocol versions gracefully degrades (realisations are unavailable). -- **Serve protocol**: - Bumped from 2.7 to 2.8 with native serialisers for the new types. - Fallback to older protocol versions gracefully degrades in the same way. - -Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. -For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. - -### Local Store SQLite schema - -The build trace entries no longer have any foreign key store objects in the store. -This is because we will need to remember the build trace entries for resolved derivations we may have deleted, otherwise we will effectively forget outputs resolved derivations we do have on disk. -GC for build trace will be implemented later --- there is no single correct choice (there is no closure property) so it will be a question of what policies users want. - -### Structured signatures - -[Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. -`nix path-info --json --json-format 3` opts into the new version for this command. -JSON parsing accepts both the old string format and new structured format for backwards compatibility. - -### Impact - -- **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. -- **Binary cache operators**: - Binary caches serving CA derivation build traces will need to be repopulated. - Existing NARs and narinfo files are unaffected. -- **Tool authors**: - Implementations interfacing with the CA derivations protocol are simplified. - The derivation hash modulo algorithm is no longer required to form build trace keys. diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md deleted file mode 100644 index fc24d5e08b9b..000000000000 --- a/doc/manual/rl-next/closure-gc.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" -issues: 7239 -prs: 15236 ---- - -`nix store delete --recursive --skip-alive` can be used to collect garbage -within a closure, in which case it will only collect the dead paths that are -part of the closure of its arguments. diff --git a/doc/manual/rl-next/filetransfer-retry-backoff.md b/doc/manual/rl-next/filetransfer-retry-backoff.md deleted file mode 100644 index 0e26ccd54a8c..000000000000 --- a/doc/manual/rl-next/filetransfer-retry-backoff.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -synopsis: Configurable file-transfer retry backoff with full jitter and Retry-After support -issues: [15419, 15023] -prs: [15449] ---- - -File transfer retries (downloads and uploads) now use AWS-style "full jitter" -exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), -and honor the `Retry-After` response header. Retry timing is configurable via -new `nix.conf` settings: - -- `filetransfer-retry-delay` (default 100ms): base delay for transient errors -- `filetransfer-retry-delay-rate-limited` (default 5000ms): base delay for 429/503 -- `filetransfer-retry-max-delay` (default 60000ms): per-attempt delay ceiling -- `filetransfer-retry-jitter` (default true): enable full jitter - -The existing `download-attempts` setting has been renamed to -`filetransfer-retry-attempts` to reflect that it applies to uploads as well as -downloads. The old name remains as an alias for backwards compatibility. - -Per-substituter overrides are available as store URL parameters -(`retry-delay`, `retry-delay-rate-limited`, `retry-max-delay`, -`retry-attempts`), e.g. `s3://my-cache?retry-attempts=8`. - -This addresses thundering-herd scenarios where many CI jobs hit the same -S3 prefix and receive 503 SlowDown; previously the retry window for 503 -was only ~4 seconds. diff --git a/doc/manual/rl-next/fix-primop-eval-state.md b/doc/manual/rl-next/fix-primop-eval-state.md deleted file mode 100644 index 1a3bc287538f..000000000000 --- a/doc/manual/rl-next/fix-primop-eval-state.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "C API: Fix `EvalState` pointer passed to primop callbacks" -prs: [15300, 15383] ---- - -The `EvalState *` passed to C API primop callbacks was incorrectly pointing to -the internal `nix::EvalState` rather than the C API wrapper struct. This caused -a segfault when the callback used the pointer with C API functions such as -`nix_alloc_value()`. The same issue affected `printValueAsJSON` and -`printValueAsXML` callbacks on external values. diff --git a/doc/manual/rl-next/getflake-path.md b/doc/manual/rl-next/getflake-path.md deleted file mode 100644 index 2360fe7693e0..000000000000 --- a/doc/manual/rl-next/getflake-path.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: "`builtins.getFlake` now supports path values" -prs: [15290] ---- - -`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref. diff --git a/doc/manual/rl-next/git-url-scp.md b/doc/manual/rl-next/git-url-scp.md deleted file mode 100644 index b1125f7b32a7..000000000000 --- a/doc/manual/rl-next/git-url-scp.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -synopsis: Support SCP-like URLs in fetchGit and type = "git" flake inputs -prs: [14863] -issues: [14852, 14867] ---- - -Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially -restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: - -```nix -builtins.fetchGit "host:/absolute/path/to/repo" -``` - -Nix also passes through the tilde (for home directories) verbatim: - -```nix -builtins.fetchGit "host:~/relative/to/home" -``` - -IPv6 addresses also supported when bracketed: - -```nix -builtins.fetchGit "user@[::1]:~/relative/to/home" -``` - -`builtins.fetchTree` also supports this syntax now: - -```nix -builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } -``` diff --git a/doc/manual/rl-next/github-fetcher-param-validation.md b/doc/manual/rl-next/github-fetcher-param-validation.md deleted file mode 100644 index 2fb430ae4b24..000000000000 --- a/doc/manual/rl-next/github-fetcher-param-validation.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: GitHub fetcher now validates URL parameters -prs: [15331] -issues: [15304] ---- - -The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. diff --git a/doc/manual/rl-next/mimalloc.md b/doc/manual/rl-next/mimalloc.md deleted file mode 100644 index dfcde0cab03a..000000000000 --- a/doc/manual/rl-next/mimalloc.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -synopsis: "Link mimalloc for faster evaluation" -prs: [15596] ---- - -The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) -by default on non-Windows platforms, replacing glibc's malloc for all -non-GC allocations. - -This yields a **5–12% wall-clock improvement** on evaluation workloads, -ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS -configurations. - -The allocator can be disabled at build time with `-Dmimalloc=disabled` -or by passing `withMimalloc = false` to the Nix package. diff --git a/doc/manual/rl-next/s3-credential-chain-web-identity.md b/doc/manual/rl-next/s3-credential-chain-web-identity.md deleted file mode 100644 index 4dfece0e3b7d..000000000000 --- a/doc/manual/rl-next/s3-credential-chain-web-identity.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -synopsis: "S3: restore STS WebIdentity and ECS container credential providers" -prs: [15507] ---- - -Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a -custom chain built on `aws-c-auth`. That chain omitted two providers, -breaking S3 binary cache access in container workloads: - -- **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, - `AWS_ROLE_SESSION_NAME`) — used by EKS IRSA, GitHub Actions OIDC, and - any `sts:AssumeRoleWithWebIdentity` federation. -- **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, - `AWS_CONTAINER_CREDENTIALS_FULL_URI`) — used by ECS tasks and EKS Pod - Identity. - -The typical symptom was a misleading IMDS error -(`Valid credentials could not be sourced by the IMDS provider`), because -IMDS is the last provider tried after the correct one was skipped. - -Both providers are now part of the chain, ordered to match the -pre-2.33 `DefaultAWSCredentialsProviderChain`: -`Environment → SSO → Profile → STS WebIdentity → (ECS | IMDS)`. -As in both the old and new AWS SDK default chains, ECS and IMDS are -mutually exclusive: when container credential environment variables are -set, IMDS is skipped. diff --git a/doc/manual/rl-next/store-config-get-state-dir.md b/doc/manual/rl-next/store-config-get-state-dir.md deleted file mode 100644 index ba628b140be7..000000000000 --- a/doc/manual/rl-next/store-config-get-state-dir.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -synopsis: "Improve daemon socket path logic for chroot stores" -prs: [15429] ---- - -The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-setting-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). -This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. - -Example: - -```bash -nix-daemon --store /foo/bar -``` - -will now use a socket at: -``` -/foo/bar/nix/var/nix/daemon-socket/socket -``` -instead of -``` -$NIX_STATE_DIR/daemon-socket/socket -``` - -Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: - -- When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: - - ```bash - nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" - ``` - -- When connecting as a client put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): - - ``` - unix://$NIX_STATE_DIR/daemon-socket/socket - ``` diff --git a/doc/manual/rl-next/zstd-multiframe.md b/doc/manual/rl-next/zstd-multiframe.md deleted file mode 100644 index 6f5d9875df74..000000000000 --- a/doc/manual/rl-next/zstd-multiframe.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -synopsis: zstd compression now emits multi-frame output and uses less memory -prs: [15550] ---- - -zstd-compressed NARs are now written as a sequence of independent 16 MiB -frames instead of a single large frame. This lays the groundwork for -parallel decompression in a future release without requiring caches to be -repopulated, and significantly lowers peak memory use during compression -(e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). - -The output remains standard zstd and is decoded unchanged by existing Nix -binaries and the `zstd` CLI; compression ratio is effectively unchanged. - -Per-frame compression now uses up to 4 worker threads. For zstd this is the -new default: the `parallel-compression` store setting defaults to `true` when -`compression=zstd` (it remains `false` for `xz`). Set -`?parallel-compression=false` to opt out. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b9020..ae77759f5516 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -19,9 +19,10 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Exposing in OS File Systems](store/file-system-object/os-file-system.md) - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - - [Store Path](store/store-path.md) + - [Store Path and Store Directory](store/store-path.md) - [Store Derivation and Deriving Path](store/derivation/index.md) - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) @@ -137,7 +138,9 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - - [Nix Cache Info Format](protocols/nix-cache-info.md) + - [Binary Cache](protocols/binary-cache/index.md) + - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) + - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Nix32 Encoding](protocols/nix32.md) - [C API](c-api.md) @@ -149,13 +152,14 @@ - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - - [JSON guideline](development/json-guideline.md) + - [Data Modeling Guidelines](development/data-modeling.md) - [C++ style guide](development/cxx.md) - [Static Analysis](development/static-analysis.md) - [Experimental Features](development/experimental-features.md) - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.35 (2026-06-22)](release-notes/rl-2.35.md) - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 07b3130f9ce9..74815df282c8 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -27,7 +27,8 @@ /contributing/documentation /development/documentation 301! /contributing/experimental-features /development/experimental-features 301! /contributing/cli-guideline /development/cli-guideline 301! -/contributing/json-guideline /development/json-guideline 301! +/contributing/json-guideline /development/data-modeling 301! +/development/json-guideline /development/data-modeling 301! /contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! @@ -36,6 +37,7 @@ /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! /language/values /language/types 301! +/language/values.html /language/types 301! /language/constructs /language/syntax 301! /language/builtin-constants /language/builtins 301! @@ -46,6 +48,7 @@ /package-management/package-management /package-management 301! /package-management/s3-substituter /store/types/s3-binary-cache-store 301! +/protocols/nix-cache-info /protocols/binary-cache/nix-cache-info 301! /protocols/protocols /protocols 301! /json/* /protocols/json/:splat 301! diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 7ea1d0e5aa99..cc7fe77eae56 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -160,7 +160,7 @@ When [`use-xdg-base-directories`] is enabled, the configuration directory is res Likewise for the state and cache directories. -## Miscellanous environment variables +## Miscellaneous environment variables - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee18..07229255e7cd 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,9 +62,9 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--max-freed`](#opt-max-freed) *bytes* - + Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 7c17ce9095b4..c1a4251b0973 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -45,20 +45,20 @@ md5sum`. - `--base32` - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix + Print the hash in [Nix32](@docroot@/protocols/nix32.md) representation rather than hexadecimal. + This representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - `--base64` - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Similar to `--base32`, but print the hash in a [Base64](https://en.wikipedia.org/wiki/Base64) representation, + which is more compact than the Nix32 one. - `--sri` - Print the hash in SRI format with base-64 encoding. + Print the hash in [SRI](@docroot@/glossary.md#gloss-sri) format with Base64 encoding. The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. + followed by a hyphen (-) and the Base64 hash body. - `--truncate` @@ -71,18 +71,18 @@ md5sum`. - `--to-base16` - Don’t hash anything, but convert the base-32 hash representation + Don’t hash anything, but convert the [Nix32](@docroot@/protocols/nix32.md) hash representation *hash* to hexadecimal. - `--to-base32` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + *hash* to [Nix32](@docroot@/protocols/nix32.md). - `--to-base64` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + *hash* to Base64. - `--to-sri` @@ -134,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal, base-32, base-64, and SRI: +Converting between hexadecimal, Nix32, Base64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 diff --git a/doc/manual/source/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md index 8451778ad46d..86c20b9e1de4 100644 --- a/doc/manual/source/command-ref/nix-prefetch-url.md +++ b/doc/manual/source/command-ref/nix-prefetch-url.md @@ -32,7 +32,7 @@ Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. -The hash is printed using base-32 unless `--type md5` is specified, +The hash is printed using [Nix32](@docroot@/protocols/nix32.md) unless `--type md5` is specified, in which case it's printed using base-16. Additionally, if the option `--print-path` is used, the path of the downloaded file in the Nix store is also printed. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 742170f76c6c..13bcd849e24f 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -199,23 +199,7 @@ Nix uses a string with the following format to identify the *system type* or *pl -[-] ``` -It is set when Nix is compiled for the given system, and based on the output of Meson's [`host_machine` information](https://mesonbuild.com/Reference-manual_builtin_host_machine.html)> - -``` ---[][-] -``` - -When cross-compiling Nix with Meson for local development, you need to specify a [cross-file](https://mesonbuild.com/Cross-compilation.html) using the `--cross-file` option. Cross-files define the target architecture and toolchain. When cross-compiling Nix with Nix, Nixpkgs takes care of this for you. - -In the nix flake we also have some cross-compilation targets available: - -``` -nix build .#nix-everything-riscv64-unknown-linux-gnu -nix build .#nix-everything-armv7l-unknown-linux-gnueabihf -nix build .#nix-everything-armv7l-unknown-linux-gnueabihf -nix build .#nix-everything-x86_64-unknown-freebsd -nix build .#nix-everything-x86_64-w64-mingw32 -``` +It is set when Nix is compiled for the given system, and based on the output of Meson's [`host_machine` information](https://mesonbuild.com/Reference-manual_builtin_host_machine.html). For historic reasons and backward-compatibility, some CPU and OS identifiers are translated as follows: @@ -232,6 +216,19 @@ For historic reasons and backward-compatibility, some CPU and OS identifiers are | `mips` | `big` | `mips` | | `mips64` | `big` | `mips64` | + +When cross-compiling Nix with Meson for local development, you need to specify a [cross-file](https://mesonbuild.com/Cross-compilation.html) using the `--cross-file` option. Cross-files define the target architecture and toolchain. When cross-compiling Nix with Nix, Nixpkgs takes care of this for you. + +In the nix flake we also have some cross-compilation targets available: + +``` +nix build .#nix-everything-riscv64-unknown-linux-gnu +nix build .#nix-everything-armv7l-unknown-linux-gnueabihf +nix build .#nix-everything-armv7l-unknown-linux-gnueabihf +nix build .#nix-everything-x86_64-unknown-freebsd +nix build .#nix-everything-x86_64-w64-mingw32 +``` + ## Compilation environments Nix can be compiled using multiple environments: diff --git a/doc/manual/source/development/json-guideline.md b/doc/manual/source/development/data-modeling.md similarity index 86% rename from doc/manual/source/development/json-guideline.md rename to doc/manual/source/development/data-modeling.md index 309b4b3a06e4..a9d131147588 100644 --- a/doc/manual/source/development/json-guideline.md +++ b/doc/manual/source/development/data-modeling.md @@ -1,7 +1,12 @@ -# JSON guideline +# Data Modeling Guidelines -Nix consumes and produces JSON in a variety of contexts. -These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. +Nix consumes and produces JSON and attribute sets in a variety of contexts. +These guidelines ensure consistent practices for our interfaces, for ease of use, and so that experience in one part carries over to another. + +For these guidelines, we will use JSON terminology, but they apply equally well to new attribute set interfaces (primops, etc.). +Note that these are guidelines first and foremost. Exceptions include: +- Feature testing: e.g., it is OK to do `builtins?frobnicate`. +- Compatibility: we generally do not change stable interfaces just to make them comply. New replacements can be added with care. ## Extensibility diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 6578632d991a..35e4c71ec388 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -26,7 +26,6 @@ or GCC. This is useful when debugging memory corruption issues. ```console [nix-shell]$ export mesonBuildType=debugoptimized [nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm -[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl [nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" ``` diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 3b80e8a266b0..6ea1a233a4aa 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -9,7 +9,7 @@ You can build it yourself: ``` # nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html +# xdg-open ./result/index.html ``` [Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. @@ -34,31 +34,28 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > │ ├── value/context.cc > │ … > │ -> ├── tests -> │ │ +> ├── libutil-tests +> │ ├── meson.build > │ … -> │ ├── libutil-tests -> │ │ ├── meson.build -> │ │ … -> │ │ └── data -> │ │ ├── git/tree.txt -> │ │ … -> │ │ -> │ ├── libexpr-test-support +> │ ├── data +> │ │ ├── git/tree.txt +> │ … +> │ +> ├── libexpr-test-support +> │ ├── meson.build +> │ ├── include/nix/expr > │ │ ├── meson.build -> │ │ ├── include/nix/expr -> │ │ │ ├── meson.build -> │ │ │ └── tests -> │ │ │ ├── value/context.hh -> │ │ │ … > │ │ └── tests -> │ │ ├── value/context.cc +> │ │ ├── value/context.hh > │ │ … -> │ │ -> │ ├── libexpr-tests -> │ … ├── meson.build +> │ ├── tests > │ ├── value/context.cc > │ … +> │ +> ├── libexpr-tests +> │ ├── meson.build +> │ ├── value/context.cc +> │ … > … > ``` @@ -257,15 +254,6 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. -### Troubleshooting - -Sometimes running tests in the development shell may leave artefacts in the local repository. -To remove any traces of that: - -```console -git clean -x --force tests -``` - ### Characterisation testing { #characterisation-testing-functional } Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. @@ -311,78 +299,34 @@ Generally, this build is sufficient, but in nightly or CI we also test the attri The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). +Because these tests are expensive and require more than what the standard github-actions setup provides, most of them only run on the master branch (on ). You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. ## Installer tests -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +GitHub Actions CI in the Nix repository also tests the installer on PRs. It does not require additional setup and utilises [GHA Artifacts](https://docs.github.com/en/actions/tutorials/store-and-share-data) and can be run in any Nix repository fork. -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: +- The `tests` job generates installers for the platforms below and uploads them as an artifact: - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command. + - `aarch64-darwin` -### One-time setup +- The `installer_test` job (which runs on Linux and macOS) will try to install Nix with the cached installer and run a trivial Nix command. +- Both the scripted installer and the [standalone Rust-based installer](https://github.com/NixOS/nix-installer) are tested. -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. +You can generate the installer tarball and script manually by running `nix build .#hydraJobs.installerScriptForGHA.`. ## Working on documentation ### Using the CI-generated installer for manual testing -After the CI run completes, you can check the output to extract the installer URL: +After the CI run completes, you can check the output to extract the installer artifact: 1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: +2. Scroll down to `Artifacts` section. +3. Download the corresponding installer artifact (`installer-darwin` for `aarch64-darwin` and `installer-linux` for `x86_64-linux`). +4. Unpack the downloaded `.zip` artifact. +5. To generate an install command, plug the path to the unpacked artifact into this template: ```console - curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + sh /install --tarball-url-prefix file:// ``` - - - diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index d7436a2052ee..188e384772b5 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -104,7 +104,7 @@ A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. - Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution). + Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md). This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. @@ -169,6 +169,11 @@ A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. +- [hermetic]{#gloss-hermetic} + + An evaluation or build process is hermetic when one can mechanically identify the set of all inputs that may affect it. + At the build level this is achieved by sandboxing; at the evaluation level by restricting impure access (as in [pure evaluation](@docroot@/command-ref/conf-file.md#conf-pure-eval)) together with [locking](#gloss-locking) or [pinning](#gloss-pinning) of the fetched inputs, taken transitively over [pure fetches](@docroot@/command-ref/conf-file.md#pure-fetch). + - [IFD]{#gloss-ifd} [Import From Derivation](./language/import-from-derivation.md) @@ -201,6 +206,10 @@ [instantiate]: #gloss-instantiate +- [locking]{#gloss-locking} + + In package management, *locking* is the concept or process of creating a lock file, which maps each mutable evaluation input to an immutable reference, so that future evaluations resolve to the same immutable versions rather than whatever the mutable references currently point to. + - [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix @@ -270,6 +279,16 @@ [package]: #package +- [pinning]{#gloss-pinning} + + Like [locking](#gloss-locking), but a pin only locks a single input. + A pinning solution may manage a collection of pins, + but serves the bottom-up purpose of fixing an input's reference on demand, + whereas locking implies a top down approach where all pins are "coerced" into a single place. + This "coercion" is generally achieved by means of high-level mechanisms such as programming language module systems. + Nix does not have such a restrictive module system, as even a flake can use expressions that pin or lock on their own. + It does not rely on the lock being total, but on a transitive fetching property; see [hermeticity](#gloss-hermetic). + - [profile]{#gloss-profile} A symlink to the current *user environment* of a user, e.g., @@ -375,6 +394,13 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs +- [SRI]{#gloss-sri} + + [Subresource Integrity](https://www.w3.org/TR/SRI/) (SRI) is a [W3C specification](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) for integrity metadata. + Nix uses the SRI hash format (`-`) to specify content hashes in a way that is self-describing, since the hash algorithm is part of the format. + + [SRI]: #gloss-sri + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that diff --git a/doc/manual/source/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md index 057fd444349e..e98067348ee6 100644 --- a/doc/manual/source/installation/prerequisites-source.md +++ b/doc/manual/source/installation/prerequisites-source.md @@ -1,80 +1,52 @@ # Prerequisites - - GNU Autoconf () and the - autoconf-archive macro collection - (). These are - needed to run the bootstrap script. - - - GNU Make. - - - Bash Shell. The `./configure` script relies on bashisms, so Bash is - required. - - - A version of GCC or Clang that supports C++23. - - - `pkg-config` to locate dependencies. If your distribution does not - provide it, you can get it from - . - - - The OpenSSL library to calculate cryptographic hashes. If your - distribution does not provide it, you can get it from - . - - - The `libbrotlienc` and `libbrotlidec` libraries to provide - implementation of the Brotli compression algorithm. They are - available for download from the official repository - . - - - cURL and its library. If your distribution does not provide it, you - can get it from . - - - The SQLite embedded database library, version 3.6.19 or higher. If - your distribution does not provide it, please install it from - . - - - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). - - To enable it, install - `pkgconfig` and the Boehm garbage collector, and pass the flag - `--enable-gc` to `configure`. - - - The `boost` library of version 1.66.0 or higher. It can be obtained - from the official web site . - - - The `editline` library of version 1.14.0 or higher. It can be - obtained from the its repository - . - - - The `libsodium` library for verifying cryptographic signatures - of contents fetched from binary caches. - It can be obtained from the official web site - . - - - Recent versions of Bison and Flex to build the parser. (This is - because Nix needs GLR support in Bison and reentrancy support in - Flex.) For Bison, you need version 2.6, which can be obtained from - the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex, - you need version 2.5.35, which is available on - [SourceForge](http://lex.sourceforge.net/). Slightly older versions - may also work, but ancient versions like the ubiquitous 2.5.4a - won't. - - - The `libseccomp` is used to provide syscall filtering on Linux. This - is an optional dependency and can be disabled passing a - `--disable-seccomp-sandboxing` option to the `configure` script (Not - recommended unless your system doesn't support `libseccomp`). To get - the library, visit . - - - On 64-bit x86 machines only, `libcpuid` library - is used to determine which microarchitecture levels are supported + This list and lower version bounds are maintained on best-effort basis. When in doubt, check the `meson.build` files. + + - Meson build system (). + + - Ninja (). + + - A version of GCC or Clang that supports C++23 (anything newer than Clang 19 or GCC 14 is likely to work). + + - `pkg-config` to locate dependencies. + If your distribution does not provide it, you can get it from . + + - The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + - The `libbrotlienc` and `libbrotlidec` libraries to provide implementation of the Brotli compression algorithm. + They are available for download from the official repository . + + - cURL library. + If your distribution does not provide it, you can get it from . + + - The SQLite embedded database library, version 3.6.19 or higher. + If your distribution does not provide it, please install it from . + + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce the evaluator’s memory consumption (optional). + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the option `-Dlibexpr:gc=enabled` to `meson setup`. + + - The `boost` library of version 1.87.0 or higher. + It can be obtained from the official web site . + + - The `editline` library of version 1.14.0 or higher. + It can be obtained from the its repository . + + - The `libsodium` library for verifying cryptographic signatures of contents fetched from binary caches. + It can be obtained from the official web site . + + - Recent versions of Bison and Flex to build the parser. + (This is because Nix needs C++ template support in Bison and reentrancy support in Flex.) + + - The `libseccomp` is used to provide syscall filtering on Linux. + This is an optional dependency and can be disabled passing a `-Dlibstore:seccomp-sandboxing=disabled` option to the `meson setup` command + (Not recommended unless your system doesn't support `libseccomp`). + To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library is used to determine which microarchitecture levels are supported (e.g., as whether to have `x86_64-v2-linux` among additional system types). - The library is available from its homepage - . - This is an optional dependency and can be disabled - by providing a `--disable-cpuid` to the `configure` script. - - - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and - RapidCheck are required, which are available at - and - respectively. + The library is available from its homepage . + This is an optional dependency and can be disabled by providing a `-Dlibutil:cpuid=disabled` option to `meson setup` script. + + - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at + and respectively. diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c917..4df3a37e70b9 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -10,7 +10,7 @@ as /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/ -where `b6gvzjyb2pg0…` is a unique identifier for the package that +where `q06x3jll2yfz…` is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features. @@ -49,7 +49,7 @@ builds correctly on your system, this is because you specified the dependency explicitly. This takes care of the build-time dependencies. Once a package is built, runtime dependencies are found by scanning -binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`). +binaries for the [hash parts](@docroot@/store/store-path.md#digest) of Nix store paths (such as `r8vvq9kq…`). This sounds risky, but it works extremely well. ## Multi-user support @@ -174,7 +174,7 @@ the package: ## Portability -Nix runs on Linux and macOS. +Nix runs on Linux, macOS and FreeBSD. ## NixOS diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index 67612029c8a4..cc2743dc5ca9 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -337,8 +337,8 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](@docroot@/glossary.md#gloss-sri). + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of Base32 encoding. > **Note** > diff --git a/doc/manual/source/language/builtins-prefix.md b/doc/manual/source/language/builtins-prefix.md index 8dd929be3601..ed78366b094a 100644 --- a/doc/manual/source/language/builtins-prefix.md +++ b/doc/manual/source/language/builtins-prefix.md @@ -28,6 +28,25 @@ Some built-ins are also exposed directly in the global scope: - [`toString`](#builtins-toString) - [`true`](#builtins-true) + + +> **Tip** +> +> **Should I use `builtins` or `lib`?** +> +> The built-ins are designed to be a stable interface that expressions can depend on over time, +> so that, for instance, old Nixpkgs versions continue to evaluate reproducibly. +> +> On the flip side, this means they have accumulated a few quirks that Nix is unable to change, +> but a library like Nixpkgs `lib` *can* improve, replace or deprecate those behaviors, +> because its sources are pinned where reproducibility matters. +> +> So while it is not wrong to use `builtins` directly, +> for instance in small Nixpkgs-independent projects, +> you will have a better experience using a library like `lib` as your primary source of functions, +> as it hides problematic functions, fixes up others, +> and helps you improve your code by means of future deprecations, which are still sufficiently rare. +
derivation attrs

derivation is described in diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 2403183fc2d2..50aa525acbf4 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -165,7 +165,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > > for an Autoconf-style package. - The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + The name of an output is combined with the name of the derivation to create the [name part](@docroot@/store/store-path.md#name) of the output's store path, unless it is `out`, in which case just the name of the derivation is used. > **Example** > diff --git a/doc/manual/source/meson.build b/doc/manual/source/meson.build index 294d57ad9f9c..115783c6b24a 100644 --- a/doc/manual/source/meson.build +++ b/doc/manual/source/meson.build @@ -8,10 +8,10 @@ summary_rl_next = custom_target( 'pipefail', '-c', ''' - if [ -e "@INPUT@" ]; then + if [ '@0@' = 'false' ]; then echo ' - [Upcoming release](release-notes/rl-next.md)' fi - ''', + '''.format(official_release), ], input : [ rl_next_generated, diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index e6a772213d6d..bc2cdfb27ab2 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,7 +19,7 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info @@ -28,7 +28,7 @@ WantMassQuery: 1 Priority: 30 ``` -When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/package-management/profiles.md b/doc/manual/source/package-management/profiles.md index 1d9e672a8def..53cf5061f834 100644 --- a/doc/manual/source/package-management/profiles.md +++ b/doc/manual/source/package-management/profiles.md @@ -11,8 +11,7 @@ in a directory another version might be stored in `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long strings prefixed to the directory names are cryptographic hashes (to be -precise, 160-bit truncations of SHA-256 hashes encoded in a base-32 -notation) of *all* inputs involved in building the package — sources, +precise, 160-bit truncations of SHA-256 hashes encoded in [Nix32](@docroot@/protocols/nix32.md)) of *all* inputs involved in building the package — sources, dependencies, compiler flags, and so on. So if two packages differ in any way, they end up in different locations in the file system, so they don’t interfere with each other. Here is what a part of a typical Nix diff --git a/doc/manual/source/protocols/binary-cache/index.md b/doc/manual/source/protocols/binary-cache/index.md new file mode 100644 index 000000000000..d86d307ad9ee --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/index.md @@ -0,0 +1,19 @@ +# Binary Cache + +The binary cache format is an interface designed for exposing a store over HTTP. + +A binary cache consists of: + +- A [`nix-cache-info`](./nix-cache-info.md) file at the root with remote-side configuration. +- For each [store object](@docroot@/store/store-object.md): + - A [`.narinfo`](./narinfo.md) file containing the object's [metadata](@docroot@/store/store-object.md#metadata) and a (usually relative) URL to the corresponding compressed NAR. + - A possibly-compressed [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) containing the store object's file system data. +- For every entry in the [build trace](@docroot@/store/build-trace.md), a JSON file at `build-trace-v2//.doi`: + - the path encodes the [key](@docroot@/protocols/json/build-trace-entry.md#key) + - the contents are the [value](@docroot@/protocols/json/build-trace-entry.md#value). + +The following [store types](@docroot@/store/types/index.md) use the binary cache format: + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) — served over HTTP(S) +- [Local Binary Cache Store](@docroot@/store/types/local-binary-cache-store.md) — stored on the file system +- [S3 Binary Cache Store](@docroot@/store/types/s3-binary-cache-store.md) — stored in an AWS S3 bucket diff --git a/doc/manual/source/protocols/binary-cache/narinfo.md b/doc/manual/source/protocols/binary-cache/narinfo.md new file mode 100644 index 000000000000..e2e2efac0eeb --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/narinfo.md @@ -0,0 +1,42 @@ +# `.narinfo` Format + +A `.narinfo` file contains the [metadata of a store object](@docroot@/store/store-object.md#metadata) in the [binary cache](@docroot@/protocols/binary-cache/index.md) format. +It is a simple line-oriented format where each line is a `Key: Value` pair. +Some keys (e.g. `Sig`) may appear multiple times. + +The file is named `.narinfo`, where `` is the [hash part](@docroot@/store/store-path.md#digest) of the store object's [store path](@docroot@/store/store-path.md). + +The fields correspond to those documented in the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format: + +| `.narinfo` field | JSON field | Differences | +|---|---|---| +| `StorePath` | [`path`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_path) | Full [store path](@docroot@/store/store-path.md) rather than [store path base name](@docroot@/store/store-path.md#base-name) | +| `URL` | [`url`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_url) | | +| `Compression` | [`compression`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_compression) | Defaults to `bzip2` if omitted | +| `FileHash` | [`downloadHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadHash) | String-encoded hash rather than structured | +| `FileSize` | [`downloadSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadSize) | | +| `NarHash` | [`narHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narHash) | String-encoded hash rather than structured | +| `NarSize` | [`narSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narSize) | | +| `References` | [`references`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_references) | Space-separated [store path base names](@docroot@/store/store-path.md#base-name) rather than a JSON array | +| `Deriver` | [`deriver`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_deriver) | [Store path base name](@docroot@/store/store-path.md#base-name); `unknown-deriver` instead of `null` | +| `Sig` | [`signatures`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_signatures) | May appear multiple times rather than using an array | +| `CA` | [`ca`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_ca) | String-encoded [content address](@docroot@/store/store-object/content-address.md) rather than structured | + +## Example + + + +``` +StorePath: /nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +URL: nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz?sha256=1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3 +Compression: xz +FileHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +FileSize: 4029176 +NarHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +NarSize: 34878 +References: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +Deriver: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv +Sig: asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +Sig: qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +CA: fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh +``` diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/binary-cache/nix-cache-info.md similarity index 90% rename from doc/manual/source/protocols/nix-cache-info.md rename to doc/manual/source/protocols/binary-cache/nix-cache-info.md index e8351e1cebe8..3859b2a10ac4 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/binary-cache/nix-cache-info.md @@ -1,6 +1,6 @@ -# Nix Cache Info Format +# `nix-cache-info` Format -The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/protocols/binary-cache/index.md) (e.g., `https://cache.example.com/nix-cache-info`). MIME type: `text/x-nix-cache-info` diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 523678e663e3..778614eb1602 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -26,7 +26,7 @@ Derivations are serialised in one of the following formats: When derivation is encoded to a [store object] we make the following choices: -- The store path name is the derivation name with `.drv` suffixed at the end +- The store path [name](@docroot@/store/store-path.md#name) is the derivation name with `.drv` suffixed at the end Indeed, the ATerm format above does *not* contain the name of the derivation, on the assumption that a store path will also be provided out-of-band. diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index c3a27d2a6e0b..2e825af6fb8a 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -4,7 +4,7 @@ title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). + This schema describes the JSON representation of an [entry](@docroot@/store/build-trace.md#entry) in a [build trace](@docroot@/store/build-trace.md). > **Warning** > @@ -39,8 +39,8 @@ additionalProperties: false key: title: Build Trace Key description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "key" part, refering to a derivation and output. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. + This is the "key" part, referring to a derivation and output. type: object required: - drvPath @@ -61,7 +61,7 @@ additionalProperties: false value: title: Build Trace Value description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "value" part, describing an output. type: object required: diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index e0be716ef781..e0c1ffef134f 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -109,7 +109,7 @@ $defs: type: string title: Store Directory description: | - The [store directory](@docroot@/store/store-path.md#store-directory) this store object belongs to (e.g. `/nix/store`). + The [path to the store directory](@docroot@/store/store-path.md#store-directory-path) this store object belongs within (e.g. `/nix/store`). additionalProperties: false impure: diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index f1f58c2bf1ac..b1251c7426e9 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -24,7 +24,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md) (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md index 72afe893ea24..d8da1e9952cf 100644 --- a/doc/manual/source/protocols/nix32.md +++ b/doc/manual/source/protocols/nix32.md @@ -1,6 +1,6 @@ # Nix32 Encoding -Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. +Nix32 is Nix's variant of [Base32](https://en.wikipedia.org/wiki/Base32) encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. ## Alphabet diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 1aa79615d1c8..bdce19d1d62c 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -18,11 +18,9 @@ where - `name` = the name of the store object. -- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) +- `store-dir` = the [path of the store directory](@docroot@/store/store-path.md#store-directory-path) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. - - Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). +- `digest` = [Nix32](@docroot@/protocols/nix32.md) representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index b358a0fdc3c3..92e5f4599440 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). + The JSON format for derivations has been slightly revised to better conform to our [data modeling guidelines](@docroot@/development/data-modeling.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + the [data modeling guidelines](@docroot@/development/data-modeling.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/source/release-notes/rl-2.30.md b/doc/manual/source/release-notes/rl-2.30.md index 34d3e5bab4c6..5a65ed99af29 100644 --- a/doc/manual/source/release-notes/rl-2.30.md +++ b/doc/manual/source/release-notes/rl-2.30.md @@ -13,7 +13,7 @@ - Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220) The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`]. - However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. + However, by exploiting how structured attrs are implemented, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix. [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md index 5d90da0c9ebd..c59ecd6c2456 100644 --- a/doc/manual/source/release-notes/rl-2.32.md +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -8,7 +8,7 @@ - Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) - Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra parameter (the store directory). We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed697029389..1ad0cf3f0564 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -135,7 +135,7 @@ This is the legacy format, preserved for backwards compatibility: ### Version 2 (`--json-format 2`) -The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes: +The new structured format follows the [data modeling guidelines](@docroot@/development/data-modeling.md) with the following changes: - **Nested structure with top-level metadata**: @@ -149,9 +149,9 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js } ``` - The map from store path base names to store object info is nested under the `info` field. + The map from [store path base names](@docroot@/store/store-path.md#base-name) to store object info is nested under the `info` field. -- **Store path base names instead of full paths**: +- **[Store path base names](@docroot@/store/store-path.md#base-name) instead of full paths**: Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths. Combined with `storeDir`, the full path can be reconstructed. diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index 335e88ee89a9..473f351907c3 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -119,7 +119,7 @@ New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket. It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`. - The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting. + The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-local-store-use-roots-daemon) store setting. This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store). diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md new file mode 100644 index 000000000000..a712b6385627 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -0,0 +1,414 @@ +# Release 2.35.0 (2026-06-22) + +## Highlights + +- Sources are copied to the store more lazily [#3121](https://github.com/NixOS/nix/issues/3121) [#15711](https://github.com/NixOS/nix/pull/15711) [#15920](https://github.com/NixOS/nix/pull/15920) + + Historically, flakes source trees have been eagerly fetched to and evaluated from the Nix store to ensure deterministic and hermetic evaluation, even if the resulting store object is not used as a derivation input. This made the implementation simpler, yet made flakes unusable in large repositories and performed unnecessary writes to the store on each change to the source tree. + + Since Nix 2.32, all I/O (excluding `path:` and `hg+:`-style inputs) for reading sources during evaluation has been funneled to their original filesystem location (or to the `~/.cache/nix/tarball-cache-v2` bare git repository for tarball-based inputs). However, the source tree was still fetched to the store -- primarily for computing the resulting content-addressed store path. In most cases, (such as importing the `nixpkgs` package set) this is not necessary. + + Touching (and hashing the NAR serialisation of) the whole source tree is unavoidable, since: + + - In case of flake inputs, `narHash` integrity must be checked eagerly. + - The `outPath` attribute of a flake must be known in advance, and for backwards compatibility must be a content-addressed store path string with [constant string context](@docroot@/language/string-context.md#string-context-constant) representing the flake source tree. + + Even within the constraints imposed by backwards compatibility requirements, there are several improvements that are achievable. To reduce the number of copies performed, Nix now hashes the input without copying first, assuming that the `.outPath` will not end up in a derivation attribute and thus would never have to be actually fetched to the store. This comes at the slight cost of doing more work in case the assumption is wrong, but results in less work in typical use cases. The evaluator continues to behave as if the copy was performed: + + - Flakes are still evaluated from the store, from the evaluator's point of view. + - `toString ./.` continues to produce a content-addressed store path string without context. + - Path resolution crossing trees located in the filesystem and in Nix's view of it (with "virtual" overlays on top) continues to work. For example, the flake source tree can contain a relative symlink pointing outside its corresponding store object (though such usage is discouraged and makes further improvements to laziness intractable). + - Reading files from the flake's `outPath` continues to work. For example, such code is well-formed and is not considered [IFD](@docroot@/language/import-from-derivation.md): + + ```nix + builtins.readFile ( /. + (builtins.unsafeDiscardStringContext self.outPath) + "/flake.nix" ) + ``` + + Similar treatment has been applied to `builtins.fetchTarball`, which no longer eagerly copies paths to the store. + `builtins.storePath` now also short-circuits on "lazy-ish" store paths and doesn't substitute unless necessary. + + This change is expected to significantly reduce disk usage required for typical evaluations and results in ~2x speedup for fetching and unpacking a nixpkgs tarball (either via `fetchTree`/flakes or via `fetchTarball`). + +- Support FreeBSD `libjail` based sandboxing, add `x86_64-freebsd` to installer [#9968](https://github.com/NixOS/nix/pull/9968) [#13281](https://github.com/NixOS/nix/pull/13281) [#15673](https://github.com/NixOS/nix/pull/15673) + + The FreeBSD build of Nix now supports build sandboxing via FreeBSD jails and is enabled by default. + A FreeBSD build has been added to the traditional installer script. The beta rust-based installer is not yet supported. + FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages and performing other tasks expected of Nix on Linux. + +## Improvements + +- HTTP/3 (QUIC) support [#15961](https://github.com/NixOS/nix/pull/15961) + + Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 (QUIC), controlled by a new [`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by default). + When enabled, Nix requests HTTP/3 and transparently falls back to HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. + The setting only takes effect when linked against a `libcurl` built with HTTP/3 support, otherwise it is ignored and Nix keeps using HTTP/2 without warning or error. + + Enable it with: + + ``` + nix.conf: http3 = true + CLI: --http3 + ``` + + Or disable with: + + ``` + nix.conf: http3 = false + CLI: --no-http3 + ``` + +- Link mimalloc for faster evaluation [#15596](https://github.com/NixOS/nix/pull/15596) + + The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. + This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. + The allocator can be disabled at build time with `-Dmimalloc=disabled`. + +- The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) + + `revCount` and `lastModified` attributes passed to the Git fetcher are no longer eagerly validated when explicitly specified. + + When not explicitly specified, `revCount` is now also a thunk value and not computed eagerly. This delays this (potentially) expensive computation until the value is actually required. + +- Configurable file-transfer retry backoff with full jitter and `Retry-After` support [#15023](https://github.com/NixOS/nix/issues/15023) [#15419](https://github.com/NixOS/nix/issues/15419) [#15449](https://github.com/NixOS/nix/pull/15449) + + File transfer retries (downloads and uploads) now use AWS-style "full jitter" exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), + and honor the `Retry-After` response header. + + Retry timing is configurable via new `nix.conf` settings: + + - [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay): base delay for transient errors + - [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited): base delay for 429/503 + - [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay): per-attempt delay ceiling + - [`filetransfer-retry-jitter`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-jitter): enable full jitter + + The existing `download-attempts` setting has been renamed to [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) to reflect that it applies to uploads as well as downloads. + The old name remains as an alias for backwards compatibility. + + Per-substituter overrides are available as store reference parameters ([`retry-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay), [`retry-delay-rate-limited`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay-rate-limited), [`retry-max-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-max-delay), [`retry-attempts`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-attempts)), e.g. `s3://my-cache?retry-attempts=8`. + +- Improve daemon socket path logic for chroot stores [#15190](https://github.com/NixOS/nix/pull/15190) + + The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). + This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. + + Example: + + ```bash + nix-daemon --store /foo/bar + ``` + + will now use a socket at: + ``` + /foo/bar/nix/var/nix/daemon-socket/socket + ``` + instead of + ``` + $NIX_STATE_DIR/daemon-socket/socket + ``` + + Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: + + - When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: + + ```bash + nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" + ``` + + - When connecting as a client, put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): + + ``` + unix://$NIX_STATE_DIR/daemon-socket/socket + ``` + +- Linux sandbox: also block `listxattr` syscalls [#15743](https://github.com/NixOS/nix/pull/15743) + + The Linux sandbox now also returns `ENOTSUP` for `listxattr`, `llistxattr` and `flistxattr`, matching the existing treatment of `getxattr`/`setxattr`/`removexattr`. + This prevents host xattrs (e.g. `security.selinux`) from leaking into builds and fixes tools such as `mkfs.ubifs` that probe xattr support via `listxattr`. + +- Support SCP-like URLs in fetchGit and type = "git" flake inputs [#14852](https://github.com/NixOS/nix/issues/14852) [#14867](https://github.com/NixOS/nix/issues/14867) [#14863](https://github.com/NixOS/nix/pull/14863) + + Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially + restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: + + ```nix + builtins.fetchGit "host:/absolute/path/to/repo" + ``` + + Nix also passes through the tilde (for home directories) verbatim: + + ```nix + builtins.fetchGit "host:~/relative/to/home" + ``` + + IPv6 addresses also supported when bracketed: + + ```nix + builtins.fetchGit "user@[::1]:~/relative/to/home" + ``` + + `builtins.fetchTree` also supports this syntax now: + + ```nix + builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } + ``` + +- `nix flake check` now supports `--print-out-paths` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) and `--out-link` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) defaulting to not creating out links if the flag is not specified. + +- Added `--skip-alive` (and `--skip-live` alias for compatibility with Lix users) option to `nix store delete` for collecting garbage within a closure [#7239](https://github.com/NixOS/nix/issues/7239) [#15236](https://github.com/NixOS/nix/pull/15236) [#15727](https://github.com/NixOS/nix/pull/15727) + + `nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. + The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. + +- `builtins.getFlake` now supports path values [#15290](https://github.com/NixOS/nix/pull/15290) + + `builtins.getFlake` now accepts path values in addition to flakerefs. This improves the usability of relative flakes, allowing you to write `builtins.getFlake ./subflake`. + This change does not allow specifying paths that are not already in the store (though they do not have valid store objects, i.e. this will not force a copy if the flake has only been hashed -- and not copied to the store). This may change in a future release. + +- `nix-profile.fish` and `nix-profile-daemon.fish` now use `$NIX_LINK` for computing the value of `NIX_PROFILE` instead of `$HOME/.nix-profile` [#14293](https://github.com/NixOS/nix/pull/14293) + +- `nix` binary now exports symbols from C bindings [#15696](https://github.com/NixOS/nix/pull/15696) + + This allows Nix plugins written against the C API to look up symbols dynamically without linking to corresponding `libnix*c.so` libraries. + +- The computed Git LFS endpoint URLs have been fixed to follow the spec [#15891](https://github.com/NixOS/nix/pull/15891) and memory usage of LFS fetches has been decreased [#15912](https://github.com/NixOS/nix/pull/15912) + +- We now verify that fetched Git LFS objects have the same OID as requested [#15845](https://github.com/NixOS/nix/pull/15845) + +- Primop documentation now includes time complexity information [#14554](https://github.com/NixOS/nix/pull/14554) + +- Improved documentation on store paths and derivation building [#14699](https://github.com/NixOS/nix/pull/14699) + +- The [build hook](@docroot@/command-ref/conf-file.md#conf-build-hook) is now killed with `SIGTERM` instead of `SIGKILL` [#15105](https://github.com/NixOS/nix/pull/15105) + +- Download/upload logs strip `userinfo` URL components [#15715](https://github.com/NixOS/nix/pull/15715) + +## Content-addressed derivations changes + + The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. + This changes the binary cache endpoints for realisations and the daemon/nix-serve protocol (gated behind a daemon protocol feature flag). + +- Realisations keyed by store path instead of hash modulo [#11897](https://github.com/NixOS/nix/issues/11897) [#12464](https://github.com/NixOS/nix/pull/12464) + + Previously, a build trace entry (realisation) was keyed by the hash modulo of the derivation. In simpler terms, derivations transitively depending on distinct fixed-output derivations with the same `outPath` would share a realisation. + + Now, build trace entries are keyed by the regular derivation store path (`.drvPath`) plus the output name. For example, instead of: + + ``` + sha256:ba7816bf8f01...!out + ``` + + The key is now: + + ``` + /nix/store/abc...-foo.drv^out + ``` + +- Removed support for "deep" realisations [#15289](https://github.com/NixOS/nix/pull/15289) + + Previously the build trace (set of "realisations") contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. + Now, it contains entries exclusively for resolved derivations. + For now, unresolved derivations will be resolved from these underlying build trace entries. + This is slower, but has the benefit of making build trace entries stateless and self-describing --- making sharing realisations easier between stores. + + This change necessitates changes to the binary cache format: + + - The directory for build traces moved from `realisations/` to `build-trace-v2/`. + - File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. + - The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. + - The deprecated `dependentRealisations` field has been removed. + + The build trace entries stored in the local SQLite database no longer have any foreign key references to store objects. + This is because the build trace entries for resolved derivations that may have been deleted need to be preserved, otherwise the outputs of other unresolved derivations will be effectively forgotten. + GC for the build trace is not yet implemented due to the lack of a clear default policy. + +- Structured signature for realisations and `path-info` [#15009](https://github.com/NixOS/nix/pull/15009) + + [Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. + `nix path-info --json --json-format 3` opts into the new version for this command. + JSON parsing accepts both the old string format and new structured format for backwards compatibility. + + This format is also used for the build trace entries in binary caches. + +- `nix realisation` command has been renamed to `nix store build-trace` [#16000](https://github.com/NixOS/nix/pull/16000) [#15948](https://github.com/NixOS/nix/pull/15948) + +## Build performance improvements + +- Make post-build-hook asynchronous [#15406](https://github.com/NixOS/nix/issues/15406) [#15451](https://github.com/NixOS/nix/pull/15451) + + The [`post-build-hook`](@docroot@/command-ref/conf-file.md#conf-post-build-hook) now runs asynchronously, without blocking the build event loop. + Dependent builds are not started until the hook finishes, but multiple hook instances are now launched concurrently -- up to the [`max-jobs`](@docroot@/command-ref/conf-file.md#conf-max-jobs) limit. + +- zstd compression now emits multi-frame output and uses less memory [#15550](https://github.com/NixOS/nix/pull/15550) + + zstd-compressed NARs are now written as a sequence of independent 16 MiB frames instead of a single large frame. + This lays the groundwork for parallel decompression in a future release without requiring caches to be repopulated, and significantly lowers peak memory use during compression + (e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + + The output remains standard zstd and is decoded unchanged by existing Nix binaries and the `zstd` CLI; compression ratio is effectively unchanged. + + Per-frame compression now uses up to 4 worker threads. For zstd this is the new default: the [`parallel-compression`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-parallel-compression) store setting defaults to `true` when `compression=zstd` (it remains `false` for other compression algorithms like `xz`). + Set `?parallel-compression=false` to opt out. + +- More parallelism for binary cache uploads [#15957](https://github.com/NixOS/nix/pull/15957) + + Uploads of NARs now start without waiting for all references to be uploaded. + Also, NARs are now uploaded in order of descending (decompressed) NAR size. + The closure invariant is still maintained by copying `.narinfo` in a topologically sorted order. + +- The derivation build scheduler memory usage reduction and performance improvements [#15611](https://github.com/NixOS/nix/pull/15611) [#15695](https://github.com/NixOS/nix/pull/15695) + + Memory usage of the derivation build scheduler has been improved to allow more state sharing. + Inefficiencies leading to quadratic complexity of scheduling build/substitution jobs have been addressed. + Scheduling resources are allocated more sparingly and freed earlier to reduce peak consumption. + + These improvements amount to ~2-8x less `nix-daemon` memory usage for typical workloads and more in larger derivation graphs, not accounting for short-lived allocations used during substitution. + + Notably, the current architecture of the build scheduler gets proportionally slower on Linux with larger heaps as derivation "builder" processes are `fork`-ed directly from the Nix process, which blocks the builder event loop for the duration of the `fork`. Thus, smaller heap of `nix-daemon` translates into faster build startups. + +- Concurrent path substitutions and eval-time fetches of the same inputs now run only once [#15555](https://github.com/NixOS/nix/pull/15555) [#15644](https://github.com/NixOS/nix/pull/15644) + + This avoids redundant work in case multiple Nix processes try to substitute/download the same resource concurrently. + +- `.narinfo` lookups in binary caches are more concurrent + + Querying the existence and path metadata in binary caches is now more asynchronous. Operations like `nix path-info` on large closures are faster and more efficient. + The build scheduler event loop now doesn't block on `.narinfo` queries, which improves performance with passthru binary caches. + +## Bug fixes + +- Fix hash collision between store paths with self-references and their zeroed-out equivalents [#15837](https://github.com/NixOS/nix/issues/15837) [#15931](https://github.com/NixOS/nix/pull/15931) + + When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. + The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. + + This release restores hashing the positions of self-references. + As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. + This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to the internal `nix::EvalState` rather than the C API wrapper struct. + This caused a segfault when the callback used the pointer with C API functions such as `nix_alloc_value()`. + The same issue affected `printValueAsJSON` and `printValueAsXML` callbacks on external values. + +- GitHub fetcher now validates URL parameters [#15304](https://github.com/NixOS/nix/issues/15304) [#15331](https://github.com/NixOS/nix/pull/15331) + + The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. + +- Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands [#15776](https://github.com/NixOS/nix/pull/15776) + + Setting [`keep-derivations`](@docroot@/command-ref/conf-file.md#conf-keep-derivations) to `true` and trying to delete a derivation with realised outputs would previously fail. + Same with [`keep-outputs`](@docroot@/command-ref/conf-file.md#conf-keep-outputs) and trying to delete an output that still has derivers. + These options no longer affect the deletion commands, and are now documented as such. + +- S3 substituters fall back to the URL's region for STS WebIdentity auth [#15594](https://github.com/NixOS/nix/pull/15594) + + When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL as a fallback for the STS endpoint region if neither `AWS_REGION` nor `AWS_DEFAULT_REGION` is set. + Previously, IRSA setups that exported `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail with a misleading "IMDS provider" error. + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a custom chain built on `aws-c-auth`. + That chain omitted two providers, breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, `AWS_ROLE_SESSION_NAME`) -- used by EKS IRSA, GitHub Actions OIDC, and any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, `AWS_CONTAINER_CREDENTIALS_FULL_URI`) -- used by ECS tasks and EKS Pod Identity. + + The typical symptom was a misleading IMDS error (`Valid credentials could not be sourced by the IMDS provider`), because IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the pre-2.33 behaviour. + As in both the old and new AWS SDK default chains, ECS and IMDS are mutually exclusive: when container credential environment variables are set, IMDS is skipped. + +- HTTP 401 and 407 responses from binary caches are no longer treated as missing files [#15877](https://github.com/NixOS/nix/pull/15877) + + Nix no longer treats `Unauthorized` and `Proxy Authentication Required` HTTP codes as an indication of a missing file. This used to be the case because AWS S3 returns 403 `Forbidden` for missing objects in unlistable buckets. 401/407 were accidentally included and this workaround is now tightly scoped to 403 responses. + +- Fixed `nixbld` gid in `/etc/group` in the Linux build sandbox when user namespaces are not supported [#15131](https://github.com/NixOS/nix/pull/15131) + +- Store garbage collection is now more robust [#15992](https://github.com/NixOS/nix/pull/15992) [#15720](https://github.com/NixOS/nix/pull/15720) [#15616](https://github.com/NixOS/nix/pull/15616) + +- Fixed deadlock for hash-mismatching fixed-output derivations [#15874](https://github.com/NixOS/nix/pull/15874) + +- `nix-copy-closure` no longer ignores `--include-outputs` flag [#15896](https://github.com/NixOS/nix/pull/15896) + +- Fixes to `recursive-nix` experimental feature + + Prior to this release, internal datastructures used to implement this feature were not used in a thread-safe manner. + Threads handling daemon connections are now reaped promptly, fixing resource leaks. + +## Contributors + +This release was made possible by the following 59 contributors: + +- Michael Wang [**(@zwang20)**](https://github.com/zwang20) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Reuben Gardos Reid [**(@ReubenJ)**](https://github.com/ReubenJ) +- StepBroBD [**(@stepbrobd)**](https://github.com/stepbrobd) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Tom [**(@thunze)**](https://github.com/thunze) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- steveoliphant [**(@steveoliphant)**](https://github.com/steveoliphant) +- espes [**(@espes)**](https://github.com/espes) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- sander [**(@sandydoo)**](https://github.com/sandydoo) +- Erik Jensen [**(@rkjnsn)**](https://github.com/rkjnsn) +- Cameron Will [**(@cwill747)**](https://github.com/cwill747) +- Maciej Krüger [**(@mkg20001)**](https://github.com/mkg20001) +- Dror Speiser [**(@drorspei)**](https://github.com/drorspei) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- Audrey Dutcher [**(@rhelmot)**](https://github.com/rhelmot) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- TyIsI [**(@TyIsI)**](https://github.com/TyIsI) +- Adam Kliś [**(@BonusPlay)**](https://github.com/BonusPlay) +- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- ryota2357 [**(@ryota2357)**](https://github.com/ryota2357) +- LIN, Jian [**(@jian-lin)**](https://github.com/jian-lin) +- znmz [**(@znmz)**](https://github.com/znmz) +- Felix Stupp [**(@Zocker1999NET)**](https://github.com/Zocker1999NET) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- Antonio Nuno Monteiro [**(@anmonteiro)**](https://github.com/anmonteiro) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- adisbladis [**(@adisbladis)**](https://github.com/adisbladis) +- Luna Nova [**(@LunNova)**](https://github.com/LunNova) +- Riccardo Mazzarini [**(@noib3)**](https://github.com/noib3) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Dario [**(@dve00)**](https://github.com/dve00) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Paul Sbarra [**(@tones111)**](https://github.com/tones111) +- edef [**(@edef1c)**](https://github.com/edef1c) +- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Alex Ionescu [**(@aionescu)**](https://github.com/aionescu) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Pierre Penninckx [**(@ibizaman)**](https://github.com/ibizaman) +- Leonard Sheng Sheng Lee [**(@sheeeng)**](https://github.com/sheeeng) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Lennart Kolmodin [**(@kolmodin)**](https://github.com/kolmodin) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Peter Collingbourne [**(@pcc)**](https://github.com/pcc) +- Simon Žlender [**(@szlend)**](https://github.com/szlend) +- Lily Foster [**(@lilyinstarlight)**](https://github.com/lilyinstarlight) +- randomizedcoder [**(@randomizedcoder)**](https://github.com/randomizedcoder) +- Krish Jaiswal diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index a879d37d208d..cb9cb3099680 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -8,7 +8,7 @@ The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. It maps the inputs of builds to the outputs of builds. -Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. +Each *[entry]{#entry}* in the build trace maps a [derivation][derivation] to a map of [output] names to [store objects][store object]. In general the derivations used as a key should be [*resolved*](./resolution.md). A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 32e800129342..a97ffcd29cef 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -1,101 +1,277 @@ # Building -## Normalizing derivation inputs +As discussed in the [main page on derivations](./derivation/index.md): -- Each input must be [realised] prior to building the derivation in question. +> A derivation is a specification for running an executable on precisely defined input to produce one or more [store objects][store object]. + +This page describes *building* a derivation, which is to say following the instructions in the derivation to actually run the executable. +Some elements of derivations are self-explanatory. +For example, the arguments specified in the derivation really are the arguments passed to the executable. +In other cases, however, there is additional common steps performed by Nix for all derivations --- mostly for setting up the build environment and collecting the built outputs. + +The chief design consideration for the building process is *determinism*. +Conventional operating systems are typically not designed with determinism in mind. +But determinism is needed to make Nix's build caching a transparent abstraction. + +> **Explanation** +> +> For example, no one wants to slightly modify a derivation, and then find that it no longer builds for an unrelated reason, because the original derivation *also* doesn't build anymore, but the cache hit on the original derivation was hiding this. +> We want builds that succeed once to continue succeeding, to encourage fearless modification of old build recipes. +> Determinism is what enables things that once worked to keep working. + +The life cycle of a build can be broken down into 3 parts: + +1. Spawn the builder process with the proper environment, including the correct process arguments, environment variables, and file system state. + +2. Wait for the builder process to exit and collect its exit status. + Exit code 0 means success; anything else is a build failure. + (Strictly speaking, Nix detects process exit by waiting for the standard output and error streams to close. + If a builder explicitly closes these streams without exiting, Nix will kill it, and deem the build a failure. + Processes should therefore exit *without* explicitly closing those standard streams, and let the exiting of the process close them implicitly.) + + Nix also logs the standard output and error of the process, but this is just for human convenience and does not influence the behavior of the system. + (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. + In practice, Nix will store the log in `/nix/var/log/nix`) + +3. Processing the outputs. + + Traditionally, this happens only after the builder has exited: the builder process should have left behind files for each output the derivation is supposed to produce, and those files are processed to turn them into bona fide store objects. + Alternatively, the builder may send messages to Nix while it's running, creating filesystem objects and linking them to an output name. + This allows outputs to be processed concurrently during the build, allows outputs to depend on other newly created store objects, and also resolves some tricky issues with content-addressing and output-to-output references. + If the processing succeeds, the resulting store objects are associated with the derivation as (the results of) a successful build. + +Step (3) is done by Nix, either externally to the build (in the traditional case, operating on the inert data left behind after the builder has exited or been killed) or concurrently with it (in the IPC case). +Step (1) however is best described not from Nix's perspective, but from the build process's perspective. + +> **Explanation** +> +> Ultimately, what matters for determinism is what the build process can observe: what resources (files, networking, etc.) it can see, what syscalls succeed or fail, etc. +> Nix can achieve this through many different sandboxing strategies (namespaces, VMs, chroots, ...), but the process shouldn't be able to tell them apart. +> We therefore specify building from the process's perspective, not Nix's perspective, to focus on *what*, not *how*. + +## What derivations can be built + +Actually only some derivations are ready to be built. +In particular, only [*resolved*](./resolution.md) derivations can be built. +That is to say, a derivation that depends on other derivations is not ready yet to be built, because some of those other derivations might not have yet been built. +If the other derivations are indeed all built, we can witness this fact by resolving the derivation, and converting all the derivation's input references into plain store paths. + +> **Note** +> +> Note that [input-addressing](derivation/outputs/input-address.md) derivations are improperly resolved. +> As discussed on the linked page, the current input-addressing algorithm does not respect resolution-equivalence of derivations (\\(\\sim_\mathrm{Drv}\\)). +> That means that if Nix properly resolved an input-addressed derivation, the resolved derivation would have different input addresses, violating expectations. +> Nix therefore improperly resolves the derivation, keeping its original input-addressed output paths, creating an invalid derivation that is both resolved and instructed to create the outputs at the originally expected paths. + +## Environment of the builder process + +This section describes how the [`builder`](./derivation/index.md#builder) is executed. + +> **Implementation detail** +> +> Nix prevents multiple [Nix instances][Nix instance] from performing the same build at the same time, for example by acquiring exclusive file locks. + +### File system + +The builder should have access to a limited file system where only certain objects are available. +The most important exposed files are the inputs (other store objects) of the (resolved) derivation. +Additionally, some other files are exposed. + +#### Store inputs + +The builder will be run against a file system in which the [store directory][store directory path] contains the [closure] of the inputs. +In particular, consider a store that just contains this closure. +That store is exposed to the file system according to the rules specified in the [Exposing Store Objects in OS File Systems](./store-path.md#exposing) documentation. +This precisely defines the file system layout of the store that should be visible to the builder process. + +> **Note** +> +> Historically, Nix exposed *at least* the following store contents to the builder, but also arbitrarily other store objects, due to limitations around operating systems' file system virtualization capabilities, and wanting to avoid copying or moving files. +> It still can do this in so-called *unsandboxed* builds. +> +> Such builds should be considered discouraged, but one that works less badly against non-mischievous derivations than might be expected. +> This is because store paths are relatively unpredictable, so a well-behaved program is unlikely to stumble upon a store object it wasn't supposed to know about. +> +> As operating systems developed better file system primitives, the need for disabling sandboxing has lessened greatly over the years, and this trend should continue into the future. + +The outputs are expected to be created in that store directory as if they were valid store objects. +(They are just files during builder execution, but during [processing outputs](#processing-outputs) they will be turned into proper store objects.) +The [environment variables](#env-vars) for each output indicate where the builder should write them; +Nix ensures that those paths do not yet exist when the builder is run. + +> **Note** +> +> In sandboxed builds, ensuring that the outputs do not exist in the store directory is trivial. +> In unsandboxed builds, it is harder in general. +> In the worst case, the derivation is in fact rewritten so different output paths are used instead, and then the outputs are rewritten back to the intended output paths after. +> In the content-addressing case rewriting would be needed either way, but in the input-addressing case, this is a significant degradation, as the point of input addressing is to avoid rewrites by knowing output paths in advance. [realised]: @docroot@/glossary.md#gloss-realise +[closure]: @docroot@/glossary.md#gloss-closure +[store directory path]: ./store-path.md#store-directory-path + +### Other file system state + +- The current working directory of the builder process will be a fresh temporary directory. + It is initially empty when the process starts except for a few input files: -- Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. + - If [`__structuredAttrs`](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs) is enabled: `.attrs.json` (the derivation attributes as JSON) and `.attrs.sh` (a Bash-compatible rendering of the same). + The environment variables `NIX_ATTRS_JSON_FILE` and `NIX_ATTRS_SH_FILE` point to these files, respectively. -## Builder Execution {#builder-execution} + - If [`passAsFile`](@docroot@/language/advanced-attributes.md#adv-attr-passAsFile) is used (only without `__structuredAttrs`): for each attribute name listed, a file `.attr-` where `` is the [Nix32](@docroot@/protocols/nix32.md)-encoded SHA-256 hash of the attribute name. + The environment variable `Path` points to the file containing the attribute's value. -The [`builder`](./derivation/index.md#builder) is executed as follows: + In sandboxed builds, this directory is at a deterministic path inside the sandbox (controlled by the [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir) setting, default `/build`). + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for the host-side location. -- A temporary directory is created where the build will take place. The - current directory is changed to this directory. +- Basic device nodes for essential operations (null device, random number generation, standard streams as a pseudo terminal) - See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information. + (A pseudo terminal would not be strictly necessary since the standard streams are passively logging, not there to facilitate interaction. + But it is still useful to entice programs to do nicer logging with e.g. colors etc.) -- The environment is cleared and set to the derivation attributes, as - specified above. +- On Linux: Process information via `/proc` -- In addition, the following variables are set: +- Minimal user and group identity information - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. +- A loopback-only network configuration with hostname set to `localhost` - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. +> **Note** +> +> Fixed-output derivations have access to additional operating system state to facilitate communication with the outside world, such as network name resolution and TLS certificate verification. +> This is necessary because these derivations are allowed to access the network, unlike regular derivations which are fully sandboxed. - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. +### Environment variables {#env-vars} - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. +The environment is cleared and set to the derivation attributes, as +specified above. - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). +For most derivations types this must contain at least: - - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the derivation. A detailed explanation of this - behavior can be found in the - [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). +- For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. + Each output path is a concatenation of the cryptographic hash of all build inputs, the `name` attribute and the output name. + (The output name is omitted if it's `out`.) - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +In addition, the following variables are set: -- If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple [Nix instances][Nix instance] from performing the same - build at the same time. +- `NIX_BUILD_TOP` contains the path of the temporary directory for this build. -- A log of the combined standard output and error is written to - `/nix/var/log/nix`. +- Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the temporary directory. + This is to prevent the builder from accidentally writing temporary files anywhere else. + Doing so might cause interference by other processes. -- The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. +- `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. -- The temporary directory is removed (unless the `-K` option was - specified). +- `HOME` is set to `/homeless-shelter`. + (Without sandboxing, this discourages programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity.) + Usually, when `HOME` is set, it is used as the location of the home directory, even if it points to a non-existent path. + +- `NIX_STORE` is set to the path of the top-level Nix [store directory path] (typically, `/nix/store`). + +- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` is set to `true` for the derivation. + A detailed explanation of this behavior can be found in the [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). + +### Arguments + +The builder is passed the arguments specified by the derivation attribute `args`. ## Processing outputs -If the builder exited successfully, the following steps happen in order to turn the output directories left behind by the builder into proper store objects: +There are two methods for processing outputs. +But first, let us cover the requirements common to both methods. -- **Normalize the file permissions** +Regardless of which method is used, each output must be turned into a valid store object. +This involves two steps: - Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Any possible `setuid` and `setgid` - bits are cleared. +- **Normalize the file permissions** - > **Note** - > - > Setuid and setgid programs are not currently supported by Nix. - > This is because the Nix archives used in deployment have no concept of ownership information, - > and because it makes the build result dependent on the user performing the build. + The files must conform to the model described in the [Exposing in OS file systems](./file-system-object/os-file-system.md) section. + For example, timestamps and permissions are canonicalised. - **Calculate the references** - Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Nix scans each output path for [references] to input store objects by looking for the [digest][store path digest] of each input. + (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) + Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. + +### Traditional (post-build) processing + +With the traditional method, the builder process on exit should have left behind files for each output the derivation is supposed to produce. +The files must be processed to turn them into bona fide store objects. +If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. +The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). +(This is not a special restriction for outputs; it is true for the references of all store objects in general.) + +In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. +For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. +Scratch store paths must therefore be used instead. +Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. + +In addition to output-to-output references, rewriting is also needed to support self-references in the content-addressing case. +An output may contain its own store path digest, which is a self-reference. +Hash functions which are secure cannot allow the easy calculation of the quasi-fixed points needed to support self-references "natively", so instead we replace all would-be self-references with a sentinel value, and then rewrite the sentinel value to be the final store path digest. +Superficially, this post-hashing rewriting breaks the content address, but as the self-references are easily identified, the rewriting can be inverted to yield the original hashed data, allowing verifying the content address after all. + +At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. +Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. + +> **Implementation detail** +> +> Nix will normally clean up and remove the temporary build directory after every build, successful or unsuccessful. +> The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. +> The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. + +### Concurrent processing via IPC + +With this method, the builder communicates with Nix during the build using inter-process communication (IPC). + +> **Implementation detail** +> +> The current implementation, `builder-rpc-v0`, exposes its interface over a limited form of the Nix daemon socket. +> Builders may use it either with their own implementation of the Nix protocol, or with the `nix store add` and `nix store submit-output` commands. +> +> Derivations with `builder-rpc-v0` in their set of [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures) +> will not receive output paths in their environment, and are expected to submit all outputs with the aforementioned commands or protocol. + +Instead of leaving files behind for Nix to process after exit, the builder explicitly requests the daemon create store objects one at a time, then sends commands assigning output names to the just-created store objects. + +Scanning for references proceeds as usual for each store object creation request, but the set of potential references to be scanned is greater: it includes both all inputs (as before) and also all previously-added store objects. +This means, if output `bar` is supposed to reference output `foo`, `foo` should be created first, and `bar` second. + +All store objects being created are content-addressed (there is no support for input-addressed outputs with the IPC approach). +When a store object is created, its content address store path will be calculated by Nix and then returned in the IPC response message. +The builder then knows what store path to use in subsequent store objects in order for reference scanning to pick them up. + +This overall approach has several advantages: + +- **No Nix-side rewriting** + + For content-addressed outputs, the builder is responsible for adding outputs in reference order, using the store paths from earlier adds in later ones. + This avoids the fragile rewriting that would otherwise be needed to fix up output-to-output references described above. + The builder, unlike Nix itself, is free to leverage domain-specific knowledge to do a better job. For example it can + + - uncompress, rewrite, and then recompress man pages, to not miss references hidden by compression. + + - make sure to rewrite data that is to be signed, like Apple binaries, before signing that data, so as not to invalidate any signatures by mistake. + +- **Pipelining** - Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. - If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. + Downstream builds that only need some outputs (e.g., a "dev" or "headers" output) can start without waiting for all outputs to be ready. + Nix doesn't yet implement this, but it could and should. +The major *disadvantage* of this approach is that it doesn't yet support self-references. +Unlike acyclic output-to-output references, self-references fundamentally do require rewriting. +The output-to-output case was only a challenge in the traditional case because all the outputs were submitted simultaneously, whereas the self-reference case is fundamentally challenging because of what it means for a hash function to be secure, as described above. +Neither batched (traditional) nor serial (IPC) submission of outputs can avoid this fundamental property of secure hash functions. +We could add support for such rewriting just for self-references, as is done for the traditional post-build processing, but we haven't yet done so as the very point of the IPC approach is to free Nix from any obligation to rewrite black-box data in unsound ways. +[references]: ./store-object.md#references +[store path digest]: ./store-path.md#digest +[store object]: ./store-object.md [Nix instance]: @docroot@/glossary.md#gloss-nix-instance +[content-addressing]: ./derivation/outputs/content-address.md +[input-addressing]: ./derivation/outputs/input-address.md +[fixed content-addressing]: ./derivation/outputs/content-address.md#fixed +[floating content-addressing]: ./derivation/outputs/content-address.md#floating +[build trace]: ./build-trace.md diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index ca2ce6665b04..9b46405cdf58 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -9,7 +9,7 @@ The outputs specification is a map, from names to specifications for individual ## Output Names {#outputs} -Output names can be any string which is also a valid [store path](@docroot@/store/store-path.md) name. +Output names can be any string which is also a valid [store path name](@docroot@/store/store-path.md#name). The name mapped to each output specification is not actually the name of the output. In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. However, an output spec named "out" describes and output store object whose name is just the derivation name. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 3fd20f17d724..6df9b94961e8 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -16,7 +16,7 @@ Concretely, this would cause a "mass rebuild" whenever any fetching detail chang To solve this problem, we compute output hashes differently, so that certain output hashes become identical. We call this concept quotient hashing, in reference to quotient types or sets. -So how do we compute the hash part of the output paths of an input-addressed derivation? +So how do we compute the [hash part](@docroot@/store/store-path.md#digest) of the output paths of an input-addressed derivation? This is done by the function `hashQuotientDerivation`, shown below. First, a word on inputs. diff --git a/doc/manual/source/store/file-system-object.md b/doc/manual/source/store/file-system-object.md index 60cb3e572063..2ffad5f0ec94 100644 --- a/doc/manual/source/store/file-system-object.md +++ b/doc/manual/source/store/file-system-object.md @@ -19,7 +19,7 @@ Every file system object is one of the following: In general, Nix does not assign any semantics to symbolic links. Certain operations however, may make additional assumptions and attempt to use the target to find another file system object. - > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. + > See [the Wikipedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. File system objects and their children form a tree. A bare file or symlink can be a root file system object. diff --git a/doc/manual/source/store/file-system-object/os-file-system.md b/doc/manual/source/store/file-system-object/os-file-system.md new file mode 100644 index 000000000000..6ce21f7788c9 --- /dev/null +++ b/doc/manual/source/store/file-system-object/os-file-system.md @@ -0,0 +1,40 @@ +# Exposing File System Objects in real operating system file systems + +Nix's [file system object] data model is minimal. +All the various other bits and pieces of real world filesystem interfaces, such as [extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes), are specifically ignored to reduce our interface surface and the reproducibility issues associated with a larger interface. +In the view of Nix's developers, the types of simple, fine-grained batch jobs (typically, building software) that Nix specializes in simply don't benefit enough from that extra complexity for it to be worth the costs of supporting it. + +But to actually be used by software, file system objects need to be made available through the operating system's file system. +This is sometimes called "mounting" or "exposing" the file system object, though do note it may or may not be implemented with what the operating system calls "mounting". + +[file system object]: ../file-system-object.md + +## Metadata normalization + +File systems typically contain other metadata that is outside Nix's data model. +To avoid this other metadata being a side channel and source of nondeterminism, Nix is careful to normalize to fixed values. +For example, on Unix, the following metadata normalization occurs: + +- The creation and last modification timestamps on all files are set to Unix Epoch 1s (00:00:01 1/1/1970 UTC) + +- The group is set to the [default group](@docroot@/command-ref/conf-file.md#conf-build-users-group) + +- The Unix mode of the file to 0444 or 0555 (i.e., read-only, with execute permission enabled if the file was originally executable). + +- Any possible `setuid` and `setgid` bits are cleared. + + > **Note** + > + > `setuid` and `setgid` programs are not currently supported by Nix. + > These special file system permissions are in general a security footgun, and with data owned by different users in different stores, it would especially be a hazard when copying store objects between stores. + > + > This restriction has not proved to be onerous in practice. + > For example, NixOS uses so called setuid-wrappers which are outside the store. + +> **Explanation** +> +> As discussed before, Nix essentially shares its file system object data model with other tools like Git. +> But those tools tend to ignore this metadata in both directions --- when reading files, like Nix, but when writing files, timestamps are set organically, and the user is free to set other special permissions (`setuid`, `setgid`, sticky, etc.) however they like. +> Normalizing, and not just ignoring, this metadata is therefore what distinguishes Nix from these other tools more than the file system object data model itself. +> +> Nix's approach is motivated by deterministic building. Whereas Git can assume that humans running commands will simply ignore timestamps etc. as appropriate, understanding they are local and ephemeral, Nix aims to run software that was not necessarily designed with Nix in mind, and is unaware of whatever sandboxing/virtualization is in place. diff --git a/doc/manual/source/store/index.md b/doc/manual/source/store/index.md index f1e8f1402988..d063fc4fdc05 100644 --- a/doc/manual/source/store/index.md +++ b/doc/manual/source/store/index.md @@ -2,4 +2,33 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). +Concretely, albeit using concepts that are only defined in the rest of the chapter, a store consists of: + +- A set of [store objects][store object], the immutable file system data. + + This can also be looked at as a map from [store paths][store path] to store objects. + +- A set of [derivations][derivation], instructions for building store objects. + + This can also be looked at as a map from [store paths][store path] to derivations. + Since store paths to derivations always end in `.drv`, and store paths to other store objects never do, the two maps can also be combined into one. + Derivations can also be encoded as store objects too. + +- A [build trace], a record of which derivations have been built and what they produced. + + > **Warning** + > + > The concept of a build trace is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +There are [multiple types of Nix stores][store type] with different capabilities, such as the default one on the [local file system][local store] (`/nix/store`) or [binary caches][binary cache]. + +[store object]: ./store-object.md +[store path]: ./store-path.md +[derivation]: ./derivation/index.md +[build trace]: ./build-trace.md + +[store type]: ./types/index.md +[local store]: ./types/local-store.md +[binary cache]: ./types/http-binary-cache-store.md diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 170d6246cfe7..eb84ab84370d 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -66,3 +66,9 @@ A store can only contain a store object if it also contains all the store object > > The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. > However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. + +### Store Object Metadata {#metadata} + +[Store implementations](@docroot@/store/types/index.md) currently associate more information than described above with a store object. +Quite arguably some of this information doesn't belong here, because it conflates concerns. +For details see the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format or the [narinfo](@docroot@/protocols/binary-cache/narinfo.md) format. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 94d94ec6d4ae..282b7545a231 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -9,7 +9,7 @@ In particular, the content-addressing scheme will ensure that the digest of the - file system object graph (the root one and its children, if it has any) - references -- [store directory](../store-path.md#store-directory) +- [store directory path](../store-path.md#store-directory-path) - name of the store object, and not any other information, which would not be an intrinsic property of that store object. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 04bdfec004c2..43037f7baac6 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -1,72 +1,136 @@ -# Store Path +# Store Path and Store Directory -> **Example** -> -> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` -> -> A rendered store path +Nix's [store object] and [file system object] data models are minimal and abstract. +But to actually be used by software, store objects need to be made available through the operating system's file system. + +This is done by exposing all the store objects in a single *[store directory][store directory path]*. +Every entry in that directory is a *[store path base name]* pointing to a store object. +Store objects exposed in this way can then be referenced by *[store paths][store path]*. + +[store object]: ./store-object.md +[file system object]: ./file-system-object.md +[store path]: #store-path +[store path base name]: #base-name +[store directory path]: #store-directory-path + +## Store Path Base Name {#base-name} -Nix implements references to [store objects](./store-object.md) as *store paths*. +Nix implements references to store objects as *store path base names*. -Think of a store path as an [opaque], [unique identifier]: -The only way to obtain store path is by adding or building store objects. -A store path will always reference exactly one store object. +Think of a store path base name as an [opaque], [unique identifier]: +The only way to obtain a store path base name is by adding or building store objects. +A store path base name will always reference exactly one store object. [opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -Store paths are pairs of +Store path base names are pairs of -- A 20-byte digest for identification -- A symbolic name for people to read +- A 20-byte [digest]{#digest} for identification +- A symbolic [name]{#name} for people to read > **Example** > > - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq` > - Name: `firefox-33.1` -To make store objects accessible to operating system processes, stores have to expose store objects through the file system. +A store path base name is rendered to a string as the concatenation of -A store path is rendered to a file system path as the concatenation of - -- [Store directory](#store-directory) (typically `/nix/store`) -- Path separator (`/`) -- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) +- Digest rendered in [Nix32], a variant of [Base32] (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name > **Example** > > ``` -> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 -> |--------| |------------------------------| |----------| -> store directory digest name +> q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |------------------------------| |----------| +> digest name > ``` -Exactly how the digest is calculated depends on the type of store path. +[Nix32]: @docroot@/protocols/nix32.md +[Base32]: https://en.wikipedia.org/wiki/Base32 + +Exactly how the digest is calculated depends on the type of store object being referenced. Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). -## Store Directory - -Every [Nix store](./index.md) has a store directory. +## Store Directory Path -Not every store can be accessed through the file system. -But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). +Every [Nix store] has a store directory path. +This is an absolute, lexically canonical (not containing any `..`, `.`, or similar) path which points to the directory where all store objects are to be found. -[file system objects]: ./file-system-object.md - -This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +[Nix store]: ./index.md > **Note** > > The store directory defaults to `/nix/store`, but is in principle arbitrary. -It is important which store a given store object belongs to: +## Store Path + +A store path is the pair of a store directory path and a [store path base name]. +It is rendered to a file system path as the concatenation of + +- [Store directory path] (typically `/nix/store`) +- Path separator (`/`) +- The [store path base name] + +> **Example** +> +> ``` +> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +When we have fixed a given store, or given store directory path (that all the stores in use share), the abstract syntax for store paths and the abstract syntax for store path base names coincide: the store directory path is known from context, so only the other two fields vary from one store path to the next. + +## Exposing Store Objects in OS File Systems {#exposing} + +Not every store can be accessed through the file system. +But if the store has a file system representation, the following should be true: + +- The store directory path is canonical: no prefix of the path (i.e. path of the first *n* path segments) points to a symlink. + In other words, the store directory can be looked up from the store directory path without following any symlinks. + (This condition is a separate condition in addition to the "lexical canonicity" described above, which is a property of just the path itself. + This (regular) "canonicity" is a property about the path and the filesystem it navigates jointly.) + + > **Note** + > + > The [`allow-symlinked-store`](@docroot@/command-ref/conf-file.md#conf-allow-symlinked-store) setting can be used to relax this requirement. + +- The store directory path in fact points to a directory. + +- The store directory contains, for every store object in the store, the [file system object] of that store object at the (rendered) [store path base name]. + The permissions and other metadata for these files in the store directory is in the normal form described in [Exposing in OS file systems](./file-system-object/os-file-system.md). + +The above properties mean that the following file accesses will work. +Suppose we have a store available on the file system per the above rules, and `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` is the store path base name of a store object in that store. + +- Suppose that the store directory (path) is `/foo/bar`. + Then, `/foo/bar/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` exists and is the file system object of that store object. + +- Suppose that we don't know what the store directory path of the store is, but we do have a capability `storeDir` to the store directory on the file system. + (This would be a "file descriptor" on Unix, or a "file handle" on Windows.) + Then (using the Unix notation for this): + ``` + openat(storeDir, "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1", O_NOFOLLOW) + ``` + will succeed (so long as the file system object is not a symlink), and the yielded capability will point to the file system object of that store object. + + (The behavior for symlinks is harder to specify because of limitations in POSIX.) + +## Relocating store objects + +The inclusion of the store directory path in the full rendered store path means that the full rendered store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +(And actually, all of the currently-supported ways of computing the digest of a store path also depend on the store directory path, as described in the [specification of store path digests](@docroot@/protocols/store-path.md). +So this is also true even just for store path base names, in general.) + +It is therefore important to consider which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee referential integrity if store paths do not cross store boundaries. -Therefore one can only copy store objects to a different store if +One can only copy store objects to a different store if - The source and target stores' directories match diff --git a/doc/manual/source/store/types/index.md.in b/doc/manual/source/store/types/index.md.in index 75274d3a5097..fbdcf0111f59 100644 --- a/doc/manual/source/store/types/index.md.in +++ b/doc/manual/source/store/types/index.md.in @@ -37,7 +37,7 @@ store as follows: * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](./local-store.md#chroot) +* Otherwise, on Linux only, use the [local chroot store](@docroot@/store/types/local-store.md#chroot) `~/.local/share/nix/root`, which will be created automatically if it does not exist. diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs index e514a99777f7..40bfef7d2f8f 100644 --- a/doc/manual/theme/head.hbs +++ b/doc/manual/theme/head.hbs @@ -11,5 +11,5 @@ MathJax = { } }; - + diff --git a/flake.lock b/flake.lock index 4c0bf91927a4..7dcbdd60af35 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", "type": "github" }, "original": { @@ -39,20 +39,16 @@ "git-hooks-nix": { "inputs": { "flake-compat": [], - "gitignore": [], "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", + "lastModified": 1783008725, + "narHash": "sha256-jGiy6+sxjNWXSjp25uoJuNfyH9zBK1PEDY0lVoL4ibQ=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", + "rev": "bca82caa46d5ec0f5d422c61fb1e30bc51313cbe", "type": "github" }, "original": { @@ -63,15 +59,15 @@ }, "nixpkgs": { "locked": { - "lastModified": 1771903837, - "narHash": "sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E=", - "rev": "e764fc9a405871f1f6ca3d1394fb422e0a0c3951", + "lastModified": 1783148766, + "narHash": "sha256-H9+N+GFtsbVC8ZniHliChM7ndizxtqVZs6bnGOLM3WQ=", + "rev": "a50de1b7d8a586adc18d2395c19de7d6058e6030", "type": "tarball", - "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.4193.a50de1b7d8a5/nixexprs.tar.xz" }, "original": { "type": "tarball", - "url": "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz" + "url": "https://channels.nixos.org/nixos-26.05/nixexprs.tar.xz" } }, "nixpkgs-23-11": { diff --git a/flake.nix b/flake.nix index 89cbe93cff7f..62944b794957 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz"; + inputs.nixpkgs.url = "https://channels.nixos.org/nixos-26.05/nixexprs.tar.xz"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; @@ -16,10 +16,8 @@ # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; - inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 inputs.git-hooks-nix.inputs.flake-compat.follows = ""; - inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ @@ -114,7 +112,7 @@ config = crossSystem; } // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") { - emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine"; + emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable}/bin/wine"; }; overlays = [ (overlayFor (pkgs: pkgs.${stdenv})) @@ -328,12 +326,6 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } - // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { - # Some perl dependencies are broken on i686-linux. - # Since the support is only best-effort there, disable the perl - # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } # Add "passthru" tests // flatMapAttrs @@ -422,10 +414,6 @@ supportsCross = false; }; - "nix-perl-bindings" = { - supportsCross = false; - }; - "nix-clang-tidy-plugin" = { supportsCross = false; }; @@ -466,6 +454,9 @@ ) ) ) + // lib.optionalAttrs (self.hydraJobs.rustInstaller ? ${system}) { + rustInstaller = self.hydraJobs.rustInstaller.${system}; + } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index f742a7440023..5978d997b6f8 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -37,30 +37,17 @@ fi ''}"; }; - meson-format = - let - meson = pkgs.meson.overrideAttrs { - doCheck = false; - doInstallCheck = false; - patches = [ - (pkgs.fetchpatch { - url = "https://github.com/mesonbuild/meson/commit/38d29b4dd19698d5cad7b599add2a69b243fd88a.patch"; - hash = "sha256-PgPBvGtCISKn1qQQhzBW5XfknUe91i5XGGBcaUK4yeE="; - }) - ]; - }; - in - { - enable = true; - files = "(meson.build|meson.options)$"; - entry = "${pkgs.writeScript "format-meson" '' - #!${pkgs.runtimeShell} - for file in "$@"; do - ${lib.getExe meson} format -ic ${../meson.format} "$file" - done - ''}"; - }; - nixfmt-rfc-style = { + meson-format = { + enable = true; + files = "(meson.build|meson.options)$"; + entry = "${pkgs.writeScript "format-meson" '' + #!${pkgs.runtimeShell} + for file in "$@"; do + ${lib.getExe pkgs.meson} format -ic ${../meson.format} "$file" + done + ''}"; + }; + nixfmt = { enable = true; excludes = [ # Invalid diff --git a/maintainers/release-process.md b/maintainers/release-process.md index f8b6b6bec572..aff0088a0d93 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -81,27 +81,33 @@ release: $ git push --set-upstream origin $VERSION-maintenance ``` -* Create a jobset for the release branch on Hydra as follows: - - * Go to the jobset of the previous release - (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). - - * Select `Actions -> Clone this jobset`. - - * Set identifier to `maintenance-$VERSION`. - - * Set description to `$VERSION release branch`. - - * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. - - * Hit `Create jobset`. - -* Wait for the new jobset to evaluate and build. If impatient, go to - the evaluation and select `Actions -> Bump builds to front of - queue`. - -* When the jobset evaluation has succeeded building, take note of the - evaluation ID (e.g. `1780832` in +* Create two jobsets for the release branch on Hydra: + + `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. + `maintenance-$VERSION-release` builds only the artifacts consumed by + `upload-release`, so a release can be cut without waiting on the full + matrix. The `-release` suffix keeps the pair adjacent in Hydra's + alphabetical jobset list and lets scripts derive one name from the + other. + + * Clone the previous `maintenance-*` jobset, set identifier + `maintenance-$VERSION`, description `$VERSION release branch`, flake + URL `github:NixOS/nix/$VERSION-maintenance`. + + * Clone the previous `maintenance-*-release` jobset (or create a new + **legacy** jobset), set identifier `maintenance-$VERSION-release`, + description `$VERSION release artifacts`, Nix expression + `packaging/release-jobs.nix` in input `src`, and add input `src` of + type *Git checkout* pointing at + `https://github.com/NixOS/nix $VERSION-maintenance`. + +* Wait for the `maintenance-$VERSION-release` jobset to evaluate and + build. If impatient, go to the evaluation and select `Actions -> Bump + builds to front of queue`. The aggregate job `release` turns green + once every required artifact is available. + +* When the release jobset evaluation has succeeded building, take note of + the evaluation ID (e.g. `1780832` in `https://hydra.nixos.org/eval/1780832`). * Tag the release: @@ -174,8 +180,9 @@ release: $ git push ``` -* Wait for the desired evaluation of the maintenance jobset to finish - building. +* Wait for the desired evaluation of the `maintenance-XX.YY-release` + jobset to finish building (the `release` aggregate job is the gating + signal). * Tag the release diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 06678553e712..b618bd900d4e 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -64,7 +64,13 @@ sub fetch { #print Dumper($evalInfo); my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; -my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; +# Flake jobsets (`maintenance-X.Y`) expose the rev via the flake URL. +# The release-artifacts jobset (`maintenance-X.Y-release`) is a legacy +# jobset whose checkout is passed in as input `src`. +my $nixRev = ($flakeInfo + ? $flakeInfo->{revision} + : $evalInfo->{jobsetevalinputs}->{src}->{revision} + // $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix-everything.x86_64-linux", 'application/json')); #print Dumper($buildInfo); diff --git a/meson.build b/meson.build index bef2e6221a53..c01297fb5477 100644 --- a/meson.build +++ b/meson.build @@ -1,15 +1,12 @@ -# This is just a stub project to include all the others as subprojects -# for development shell purposes +# This is just a top-level project to include all the others as subprojects +# for development shell purposes (when building via Nix) or for distro packaging purposes. project( - 'nix-dev-shell', + 'Nix', 'cpp', version : files('.version'), subproject_dir : 'src', - default_options : [ - 'localstatedir=/nix/var', - ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', ) # Internal Libraries @@ -21,6 +18,14 @@ subproject('libflake') subproject('libmain') subproject('libcmd') +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libfetchers-c') +subproject('libexpr-c') +subproject('libflake-c') +subproject('libmain-c') + # Executables subproject('nix') @@ -37,21 +42,6 @@ if get_option('doc-gen') endif endif -# External C wrapper libraries -subproject('libutil-c') -subproject('libstore-c') -subproject('libfetchers-c') -subproject('libexpr-c') -subproject('libflake-c') -subproject('libmain-c') - -asan_enabled = 'address' in get_option('b_sanitize') - -# Language Bindings -if get_option('bindings') and not meson.is_cross_build() and not asan_enabled - subproject('perl') -endif - # Testing if get_option('unit-tests') subproject('libutil-test-support') @@ -63,7 +53,11 @@ if get_option('unit-tests') subproject('libexpr-tests') subproject('libflake-tests') endif -subproject('nix-functional-tests') + +if get_option('functional-tests') + subproject('nix-functional-tests') +endif + if get_option('json-schema-checks') subproject('json-schema-checks') endif diff --git a/meson.options b/meson.options index a306a84252ea..2e0d873fae08 100644 --- a/meson.options +++ b/meson.options @@ -15,10 +15,10 @@ option( ) option( - 'bindings', + 'functional-tests', type : 'boolean', value : true, - description : 'Build language bindings (e.g. Perl)', + description : 'Build functional (E2E) tests', ) option( diff --git a/nix-meson-build-support/common/asan-options/asan-options.cc b/nix-meson-build-support/common/asan-options/asan-options.cc index c9782fea03b5..62db7ddd20e0 100644 --- a/nix-meson-build-support/common/asan-options/asan-options.cc +++ b/nix-meson-build-support/common/asan-options/asan-options.cc @@ -1,4 +1,17 @@ -extern "C" [[gnu::retain, gnu::weak]] const char * __asan_default_options() +#if defined(__has_attribute) +# if __has_attribute(no_profile_instrument_function) +# define NIX_NO_PROFILE_INSTRUMENT_FUNCTION __attribute__((no_profile_instrument_function)) +# endif +#endif + +#ifndef NIX_NO_PROFILE_INSTRUMENT_FUNCTION +# define NIX_NO_PROFILE_INSTRUMENT_FUNCTION +#endif + +// This ASan hook is linked into many instrumented binaries and libraries. Do +// not emit coverage counters for it because repeated weak definitions of the +// hook can produce profile data that llvm-profdata treats as corrupted. +extern "C" [[gnu::retain, gnu::weak]] NIX_NO_PROFILE_INSTRUMENT_FUNCTION const char * __asan_default_options() { // We leak a bunch of memory knowingly on purpose. It's not worthwhile to // diagnose that memory being leaked for now. diff --git a/nix-meson-build-support/common/asan-options/meson.build b/nix-meson-build-support/common/asan-options/meson.build index 56e6a6a56a7f..80527b5a9884 100644 --- a/nix-meson-build-support/common/asan-options/meson.build +++ b/nix-meson-build-support/common/asan-options/meson.build @@ -1,7 +1,7 @@ # Clang gets grumpy about missing libasan symbols if -shared-libasan is not # passed when building shared libs, at least on Linux if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( - 'b_sanitize', + 'b_sanitize', )) add_project_link_arguments('-shared-libasan', language : 'cpp') endif diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index f8394f4fabd4..24f02a1fd943 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -41,8 +41,6 @@ Checks: - -bugprone-unused-local-non-trivial-variable # 2 warnings - returning const& from parameter - -bugprone-return-const-ref-from-parameter - # 1 warning - unsafe C functions (e.g., getenv) - - -bugprone-unsafe-functions # 1 warning - signed char misuse - -bugprone-signed-char-misuse # 1 warning - calling parent virtual instead of override @@ -53,11 +51,6 @@ Checks: - -bugprone-macro-parentheses # 1 warning - increment/decrement in conditions - -bugprone-inc-dec-in-conditions - # 2 warnings - std::move on forwarding reference (auto&&) in ranges lambdas - - -bugprone-move-forwarding-reference - # 2 warnings - coroutine pattern: co_await await(std::move(waitees)) then reuse. - # Relies on moved-from containers being empty (holds for libstdc++/libc++). - - -bugprone-use-after-move # 2 warnings - sorts Value* by ->string_view(), not by pointer value (false positive) - -bugprone-nondeterministic-pointer-iteration-order # 9 warnings - intentional std::bit_cast/memcpy on Value* arrays (evaluator hot path) @@ -70,7 +63,7 @@ Checks: # template; fires when T=unsigned char but that instantiation is correct. - -bugprone-unintended-char-ostream-output # - # Non-bugprone checks (also disabled to pass on current codebase): + # Non-bugprone checks (some disabled to pass on current codebase): # # 4 warnings - exceptions not derived from std::exception # All thrown exceptions must derive from std::exception @@ -79,7 +72,15 @@ Checks: # - cppcoreguidelines-pro-type-cstyle-cast # 11 warnings - coroutine lambdas with captures (intentional pattern in async goal/store code) # - cppcoreguidelines-avoid-capturing-lambda-coroutines - # Custom nix checks (when added) + - performance-noexcept-swap + - performance-noexcept-move-constructor + - performance-noexcept-destructor + - performance-use-std-move + - misc-throw-by-value-catch-by-reference + - cppcoreguidelines-missing-std-forward + - android-cloexec-open + - android-cloexec-pipe2 + # Custom nix checks - nix-* CheckOptions: @@ -90,3 +91,10 @@ CheckOptions: bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options;__wrap___assert_fail;_SingleDerivedPathRaw;_DerivedPathRaw;_SingleBuiltPathRaw;_BuiltPathRaw' # Allow explicitly discarding return values with (void) cast bugprone-unused-return-value.AllowCastToVoid: true + bugprone-unsafe-functions.ReportDefaultFunctions: false + # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. + bugprone-unsafe-functions.CustomFunctions: > + ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; + ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; + +ExtraArgs: ["-Werror=unnecessary-virtual-specifier"] diff --git a/nix-meson-build-support/common/clang-tidy/build_required_targets.py b/nix-meson-build-support/common/clang-tidy/build_required_targets.py index d55acd74a21e..24e4f290c607 100755 --- a/nix-meson-build-support/common/clang-tidy/build_required_targets.py +++ b/nix-meson-build-support/common/clang-tidy/build_required_targets.py @@ -53,8 +53,6 @@ def main(): + [t for t in custom_commands if t.endswith(".gen.inc")] # Flex/Bison generated parsers + [t for t in custom_commands if t.endswith("-tab.cc")] - # Perl XS generated bindings - + [t for t in custom_commands if t.endswith(".cc") and "perl" in t.lower()] ) ninja_build(args.build_root, targets) diff --git a/nix-meson-build-support/common/clang-tidy/clean_compdb.py b/nix-meson-build-support/common/clang-tidy/clean_compdb.py index 659667209ac3..8087b0bf9b6e 100755 --- a/nix-meson-build-support/common/clang-tidy/clean_compdb.py +++ b/nix-meson-build-support/common/clang-tidy/clean_compdb.py @@ -48,9 +48,6 @@ def cmdfilter(item: dict) -> bool: # Filter out Flex/Bison generated parsers (generated code) if file.endswith("-tab.cc"): return False - # Filter out Perl XS generated bindings (generated code) - if "/perl/" in file and file.endswith(".cc"): - return False return True return [chomp(x) for x in compdb if cmdfilter(x)] diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index edb40635f298..e9964bce2ac1 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -14,7 +14,7 @@ if host_machine.system() == 'cygwin' ) endif -add_project_arguments( +warning_flags = [ '-Wdeprecated-copy', '-Werror=suggest-override', '-Werror=switch', @@ -26,21 +26,43 @@ add_project_arguments( '-Werror=non-virtual-dtor', '-Wignored-qualifiers', '-Wimplicit-fallthrough', + # Clang complains about #embed even though it's now standard in C23. In C++ it's an extension, but meh. + '-Wno-c23-extensions', '-Wno-deprecated-declarations', + '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. + '-Wno-subobject-linkage', # GCC doesn't like unity builds. + # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` + # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently + # became `.version = {16, 0}` instead of failing, breaking protocol compatibility + # in a subtle way + '-Werror=c99-designator', +] + +if meson.project_name() in [ + 'nix-util', + 'nix-store', + 'nix-util-c', + 'nix-store-c', + 'nix-expr', + 'nix-expr-c', +] + # Catch vtables with vague linkage. All vtables must have a "key" function to make + # sure they are emitted as strong symbols (as required by Itanium ABI). This is needed + # in certain cases of dynamic linking, where weak symbols are not always coalesced. Otherwise + # typeinfo symbols get duplicated and dynamic_cast/exception handling ends up broken on Darwin's + # libc++. + # Search for 'virtual void anchor' in the repo to see how this is done. + warning_flags += [ '-Werror=weak-vtables' ] +endif + +add_project_arguments( + cxx.get_supported_arguments(warning_flags), language : 'cpp', ) # GCC doesn't benefit much from precompiled headers. do_pch = cxx.get_id() == 'clang' -if cxx.get_id() == 'gcc' - add_project_arguments( - '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. - '-Wno-subobject-linkage', # GCC doesn't like unity builds. - language : 'cpp', - ) -endif - # This is a clang-only option for improving build times. # It forces the instantiation of templates in the PCH itself and # not every translation unit it's included in. @@ -50,11 +72,6 @@ endif # instantiations in libutil and libstore. if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') - # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` - # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently - # became `.version = {16, 0}` instead of failing, breaking protocol compatibility - # in a subtle way - add_project_arguments('-Werror=c99-designator', language : 'cpp') endif # Detect if we're using libstdc++ (GCC's standard library) diff --git a/nix-meson-build-support/export/meson.build b/nix-meson-build-support/export/meson.build index 62a27bd48c37..d5cb39661d06 100644 --- a/nix-meson-build-support/export/meson.build +++ b/nix-meson-build-support/export/meson.build @@ -11,12 +11,34 @@ endforeach requires_public += deps_public extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {}) +fs = import('fs') +plugin_c_api_enabled = get_variable('plugin_c_api_enabled', false) + +if plugin_c_api_enabled + this_libraries = get_variable('this_libraries') + this_library = this_libraries.get_shared_lib() + this_static_library = this_libraries.get_static_lib() +else + this_library = get_variable('this_library') +endif extra_cflags = [] if not meson.project_name().endswith('-c') extra_cflags += [ '-std=c++23' ] endif +whole_archive_dep_name = meson.project_name() + '-whole-archive' +installed_whole_archive_link_arg = '' +if plugin_c_api_enabled and host_machine.system() == 'darwin' + installed_whole_archive_link_arg = '-Wl,-force_load,${libdir}/' + fs.name( + this_static_library.full_path(), + ) +elif plugin_c_api_enabled and not host_machine.system().startswith('windows') + installed_whole_archive_link_arg = '-Wl,--whole-archive,${libdir}/' + fs.name( + this_static_library.full_path(), + ) + ',--no-whole-archive' +endif + import('pkgconfig').generate( this_library, filebase : meson.project_name(), @@ -29,6 +51,19 @@ import('pkgconfig').generate( variables : extra_pkg_config_variables, ) +if installed_whole_archive_link_arg != '' + import('pkgconfig').generate( + filebase : whole_archive_dep_name, + name : 'Nix whole-archive static dependency', + description : 'Force-load static Nix C API archive into the host executable', + extra_cflags : extra_cflags, + requires : requires_public, + requires_private : requires_private, + libraries : [ installed_whole_archive_link_arg ], + variables : extra_pkg_config_variables, + ) +endif + meson.override_dependency( meson.project_name(), declare_dependency( @@ -39,3 +74,16 @@ meson.override_dependency( variables : extra_pkg_config_variables, ), ) + +if installed_whole_archive_link_arg != '' + meson.override_dependency( + whole_archive_dep_name, + declare_dependency( + include_directories : include_dirs, + link_whole : [ this_static_library ], + compile_args : [ '-std=c++23' ], + dependencies : deps_public_subproject + deps_public, + variables : extra_pkg_config_variables, + ), + ) +endif diff --git a/packaging/aws-c-io-s2n-darwin.patch b/packaging/aws-c-io-s2n-darwin.patch new file mode 100644 index 000000000000..e3d9bf084e08 --- /dev/null +++ b/packaging/aws-c-io-s2n-darwin.patch @@ -0,0 +1,26 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 403bbf6..f095c4d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -100,6 +100,9 @@ elseif (APPLE) + "source/posix/*.c" + "source/darwin/*.c" + ) ++ # nix#15857: use s2n TLS, not fork-unsafe Apple SecureTransport ++ list(REMOVE_ITEM AWS_IO_OS_SRC "${CMAKE_CURRENT_SOURCE_DIR}/source/darwin/secure_transport_tls_channel_handler.c") ++ set(USE_S2N ON) + + find_library(SECURITY_LIB Security) + find_library(NETWORK_LIB Network) +diff --git a/cmake/aws-c-io-config.cmake b/cmake/aws-c-io-config.cmake +index 156e032..d6b222d 100644 +--- a/cmake/aws-c-io-config.cmake ++++ b/cmake/aws-c-io-config.cmake +@@ -1,6 +1,6 @@ + include(CMakeFindDependencyMacro) + +-if (UNIX AND NOT APPLE AND NOT BYO_CRYPTO) ++if (UNIX AND NOT BYO_CRYPTO) + find_dependency(s2n) + endif() + diff --git a/packaging/components.nix b/packaging/components.nix index 112791c27d38..57bb08c4aca8 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -139,8 +139,6 @@ let !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin) # build failure && !stdenv.hostPlatform.isStatic - # LTO breaks exception handling on x86-64-darwin. - && stdenv.system != "x86_64-darwin" ) '' case "$mesonBuildType" in @@ -331,6 +329,11 @@ in */ withUnityBuild = true; + /** + Whether to embed the public C API into nix-cli so plugins can resolve those symbols from the executable. + */ + withPluginCAPI = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + /** A user-provided extension function to apply to each component derivation. */ @@ -479,7 +482,9 @@ in /** The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does. */ - nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-cli = callPackage ../src/nix/package.nix { + version = fineVersion; + }; nix-functional-tests = callPackage ../tests/functional/package.nix { version = fineVersion; @@ -512,8 +517,6 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; - # The clang-tidy plugin is a build-time tool loaded into clang-tidy itself, # so it must be built with a clang stdenv for ABI compatibility with the # clang-tidy binary from the same llvmPackages set, regardless of the diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b44f6ae46c80..f8f64abf3968 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -17,16 +17,16 @@ scope: { inherit stdenv; mimalloc = - if lib.versionAtLeast pkgs.mimalloc.version "3.3.0" then + if lib.versionAtLeast pkgs.mimalloc.version "3.3.2" then pkgs.mimalloc else pkgs.mimalloc.overrideAttrs rec { - version = "3.3.0"; + version = "3.3.2"; src = pkgs.fetchFromGitHub { owner = "microsoft"; repo = "mimalloc"; tag = "v${version}"; - hash = "sha256-xy9gPihw3xvhnd6BrCYfMnnRp5dPSodynKRToYwxuzg="; + hash = "sha256-GZ37qQVDe9jgMb4Coe5oKvgaLTspZDlSkS5rdy1MfUU="; }; }; @@ -44,24 +44,78 @@ scope: { NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576"; }); - curl = - (pkgs.curl.override { - http3Support = !pkgs.stdenv.hostPlatform.isWindows; - # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. - zstdSupport = true; - brotliSupport = true; - zlibSupport = true; - }).overrideAttrs - { - # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. - # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. - meta.broken = false; - }; + curl = pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + }; libblake3 = pkgs.libblake3.override { - useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + useTBB = + !( + stdenv.hostPlatform.isWindows + || stdenv.hostPlatform.isStatic + # Some tbb tests fail with libc++. + || (stdenv.cc.libcxx != null && stdenv.cc.libcxx.isLLVM) + ); }; + # Force the s2n TLS backend in aws-c-io on macOS; Apple SecureTransport is not + # fork-safe and crashes daemon workers (NixOS/nix#15857). Override it across + # the whole aws-c-* stack so one aws-c-io is shared. + aws-crt-cpp = + if !stdenv.hostPlatform.isDarwin then + pkgs.aws-crt-cpp + else + let + aws-c-io = pkgs.aws-c-io.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./aws-c-io-s2n-darwin.patch ]; + }); + aws-c-http = pkgs.aws-c-http.override { inherit aws-c-io; }; + aws-c-auth = pkgs.aws-c-auth.override { inherit aws-c-io aws-c-http; }; + aws-c-event-stream = pkgs.aws-c-event-stream.override { inherit aws-c-io; }; + aws-c-mqtt = pkgs.aws-c-mqtt.override { inherit aws-c-io aws-c-http; }; + aws-c-s3 = pkgs.aws-c-s3.override { inherit aws-c-io aws-c-http aws-c-auth; }; + in + pkgs.aws-crt-cpp.override { + inherit + aws-c-io + aws-c-http + aws-c-auth + aws-c-event-stream + aws-c-mqtt + aws-c-s3 + ; + }; + + sqlite = + if !stdenv.hostPlatform.isWindows then + pkgs.sqlite + else + pkgs.sqlite.overrideAttrs (prevAttrs: { + nativeBuildInputs = lib.filter (x: !(x.pname == "tcl")) prevAttrs.nativeBuildInputs or [ ]; + configureFlags = (lib.filter (x: !(lib.hasPrefix "--with-tcl" x)) prevAttrs.configureFlags) ++ [ + "--disable-tcl" + ]; + }); + + libgit2 = + if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then + pkgs.libgit2 + else + # Grab newer libgit2. + pkgs.libgit2.overrideAttrs rec { + version = "1.9.4"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + tag = "v${version}"; + hash = "sha256-ZKUiz3pdFE2SKxh53X2oyr7hs32Njj5YVA0OXDXz7h0="; + }; + }; + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { @@ -72,6 +126,9 @@ scope: { "--with-iostreams" "--with-url" ]; + patches = [ + ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch + ]; enableIcu = false; }).overrideAttrs (old: { diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index a29d65cb2a8d..d18e77c52f12 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -127,7 +127,6 @@ nixComponents.callPackage ( rest = builtins.substring 2 (builtins.stringLength flag) flag; in "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; availableComponents = lib.filterAttrs ( @@ -170,12 +169,7 @@ nixComponents.callPackage ( # perhaps other things that are primarily for overriding and not the shell. config = { # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] - ); + getComponents = c: builtins.removeAttrs c (lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]); }; /** @@ -211,7 +205,6 @@ nixComponents.callPackage ( "nix-fetchers-tests" "nix-flake-tests" "nix-functional-tests" - "nix-perl-bindings" ] (_: null)) c ); }; @@ -287,11 +280,14 @@ nixComponents.callPackage ( ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) - ) ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) + ++ map (transformFlag "libutil-c") (ignoreCrossFile nixComponents.nix-util-c.mesonFlags) + ++ map (transformFlag "libstore-c") (ignoreCrossFile nixComponents.nix-store-c.mesonFlags) + ++ map (transformFlag "libfetchers-c") (ignoreCrossFile nixComponents.nix-fetchers-c.mesonFlags) + ++ map (transformFlag "libexpr-c") (ignoreCrossFile nixComponents.nix-expr-c.mesonFlags) + ++ map (transformFlag "libflake-c") (ignoreCrossFile nixComponents.nix-flake-c.mesonFlags) + ++ map (transformFlag "libmain-c") (ignoreCrossFile nixComponents.nix-main-c.mesonFlags) ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); nativeBuildInputs = @@ -320,7 +316,7 @@ nixComponents.callPackage ( pkgs.buildPackages.gnused modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) - pkgs.buildPackages.nixfmt-rfc-style + pkgs.buildPackages.nixfmt pkgs.buildPackages.shellcheck pkgs.buildPackages.include-what-you-use ] @@ -328,7 +324,7 @@ nixComponents.callPackage ( ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( lib.hiPrio pkgs.buildPackages.clang-tools ) - ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; + ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold; in # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections # from making its way into NIX_CFLAGS_COMPILE. @@ -345,8 +341,7 @@ nixComponents.callPackage ( lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark ++ dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + ); propagatedBuildInputs = dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) diff --git a/packaging/everything.nix b/packaging/everything.nix index 751d861c9e80..74629d684036 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -41,8 +41,6 @@ nix-internal-api-docs, nix-external-api-docs, - nix-perl-bindings, - testers, patchedSrc ? null, @@ -65,16 +63,7 @@ let nix-main-c nix-cmd ; - } - // - lib.optionalAttrs - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - { - # Currently fails in static build - inherit - nix-perl-bindings - ; - }; + }; devdoc = buildEnv { name = "nix-${nix-cli.version}-devdoc"; @@ -116,7 +105,7 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = true; /** - `doCheck` controles whether tests are added as build gate for the combined package. + `doCheck` controls whether tests are added as build gate for the combined package. This includes both the unit tests and the functional tests, but not the integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). */ @@ -144,13 +133,6 @@ stdenv.mkDerivation (finalAttrs: { lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ nix-util-tests.tests.run-without-new-syscalls - ] - ++ - lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings ]; nativeBuildInputs = [ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 8f7e13c1f2ef..e30b62eb578d 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -33,7 +33,6 @@ let forAllPackages = forAllPackages' { }; forAllPackages' = { - enableBindings ? false, enableDocs ? false, # already have separate attrs for these }: lib.genAttrs ( @@ -66,9 +65,6 @@ let "nix-json-schema-checks" "nix-clang-tidy-plugin" ] - ++ lib.optionals enableBindings [ - "nix-perl-bindings" - ] ++ lib.optionals enableDocs [ "nix-manual" "nix-manual-manpages-only" @@ -85,7 +81,6 @@ rec { let arbitrarySystem = "x86_64-linux"; listedPkgs = forAllPackages' { - enableBindings = true; enableDocs = true; } (_: null); actualPkgs = lib.concatMapAttrs ( @@ -176,8 +171,6 @@ rec { # Build without unity to catch include issues. withUnityBuild = false; nix-expr = super.nix-expr.override { enableGC = false; }; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = null; } ) ); @@ -201,8 +194,6 @@ rec { pkgs.nixComponents2.overrideScope ( self: super: { withTSan = true; - # Dies at startup. - nix-perl-bindings = null; # TSan has issues with fork and threads. nix-functional-tests = super.nix-functional-tests.overrideAttrs { doCheck = false; }; } @@ -218,6 +209,8 @@ rec { tidyScope = pkgs.nixComponents2.overrideScope ( self: super: { withClangTidy = true; + # clang-tidy doesn't seem to like unity builds. + withUnityBuild = false; # nix-everything is built via callPackage (not the layer system), so # enableClangTidyLayer's doCheck=false doesn't reach it. Set it here # so checkInputs (the *-tests.tests.run derivations) aren't pulled in. @@ -254,9 +247,6 @@ rec { ) (forAllSystems (system: components.${system}.${pkgName})) ); - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); - # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. @@ -296,6 +286,32 @@ rec { } ); + # `NixOS/nix-installer` with this revision's Nix closure embedded. + rustInstaller = + lib.genAttrs + ( + linux64BitSystems + ++ [ + "x86_64-darwin" + "aarch64-darwin" + ] + ) + ( + system: + let + pkgs = nixpkgsFor.${system}.native; + # Embed the native (glibc) Nix even though the Linux installer + # binary is static/musl. + tarball = pkgs.callPackage ./rust-installer/tarball.nix { + nix = pkgs.nixComponents2.nix-everything; + }; + builder = if pkgs.stdenv.hostPlatform.isLinux then pkgs.pkgsStatic else pkgs; + in + builder.callPackage ./rust-installer { + inherit tarball; + } + ); + # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch new file mode 100644 index 000000000000..7ec13724c40b --- /dev/null +++ b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch @@ -0,0 +1,102 @@ +From 5883212311535a0046031d74d1568ae173c1e35b Mon Sep 17 00:00:00 2001 +From: Sergei Zimmerman +Date: Tue, 21 Jul 2026 21:15:51 +0000 +Subject: [PATCH] Fix uncaught_exceptions() not accounting for forced_unwind + +Unwound fibers would see std::uncaught_exceptions() == 0, while a +forced_unwind exception is in "flight". This goes against the contract +of std::uncaught_exceptions() that scope guards rely upon. Failing +to report the correct number of uncaught exceptions (especially +misreporting zero) will lead to scope guards to misbehave badly and skip +running cleanup code which branches on whether the destructor is called +during stack unwinding or not. + +This is because the "throw" would happen before the destructor is run on +the fiber stack being switched to, but the increment would be clobbered +by the destructor of manage_exception_state. + +I'm not sure what the contract of run ontop_fcontext is wrt to whether +the the caller provided function can throw or not, but in my best +understanding the forced_unwind mechanism is mostly internal and so is +throwing from ontop_fcontext in the switched-to fiber. Thus, I've kept +the catch block scoped to detail::forced_unwind. +--- + include/boost/context/fiber_fcontext.hpp | 37 +++++++++++++++++------- + test/test_fiber.cpp | 24 +++++++++++++++ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/include/boost/context/fiber_fcontext.hpp b/include/boost/context/fiber_fcontext.hpp +index 543ba6c..38476c9 100644 +--- a/boost/context/fiber_fcontext.hpp ++++ b/boost/context/fiber_fcontext.hpp +@@ -70,7 +70,9 @@ namespace context { + namespace detail { + + // manage_exception_state is a dummy struct unless we have specific support +-struct manage_exception_state {}; ++struct manage_exception_state { ++ void from_forced_unwind() noexcept {} ++}; + + } // namespace detail + } // namespace context +@@ -90,6 +92,11 @@ public: + manage_exception_state() { + exception_state_ = *__cxa_get_globals(); + } ++ // Hack to account for the forced_unwind exception thrown in fiber_unwind ++ // that's run ontop before the destructor. ++ void from_forced_unwind() noexcept { ++ exception_state_.uncaughtExceptions += 1; ++ } + ~manage_exception_state() { + *__cxa_get_globals() = exception_state_; + } +@@ -376,13 +383,18 @@ public: + BOOST_ASSERT( nullptr != fctx_); + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); +- return { detail::jump_fcontext( ++ try { ++ return { detail::jump_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- nullptr).fctx }; ++ nullptr).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + template< typename Fn > +@@ -391,14 +403,19 @@ public: + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); + auto p = std::forward< Fn >( fn); +- return { detail::ontop_fcontext( ++ try { ++ return { detail::ontop_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- & p, +- detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ & p, ++ detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + explicit operator bool() const noexcept { diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix new file mode 100644 index 000000000000..28ee0cb4e00b --- /dev/null +++ b/packaging/release-jobs.nix @@ -0,0 +1,83 @@ +# Hydra jobset containing only the artifacts consumed by +# `maintainers/upload-release.pl`, so a release can be cut without +# waiting on the full `hydraJobs` CI matrix. +# +# Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake +# jobsets to `outputs.hydraJobs`; we re-enter the flake via +# `builtins.getFlake` so derivations stay identical to the flake jobset +# and share builds through the binary cache. +# +# Hydra jobset configuration: +# Identifier: maintenance--release +# Type: Legacy +# Nix expression: packaging/release-jobs.nix in input `src` +# Inputs: +# src (Git checkout) https://github.com/NixOS/nix +{ + src ? { + outPath = ./..; + }, +}: +let + # Fetch by GitHub ref rather than the bare store path Hydra hands us, + # so `rev`/`lastModified` (and thus the version suffix) match the flake + # jobset and derivations are shared. + flake = builtins.getFlake ( + if src ? rev then + "github:NixOS/nix/${src.rev}" + else + # Local evaluation / testing. + builtins.unsafeDiscardStringContext (toString src) + ); + inherit (flake) hydraJobs; + inherit (flake.inputs.nixpkgs) lib; + + jobs = { + # `nix-everything` per system: provides the store paths for + # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via + # its `doc` output. + build.nix-everything = hydraJobs.build.nix-everything; + buildCross.nix-everything = { + # Only the cross targets that end up in `fallback-paths.nix`. + inherit (hydraJobs.buildCross.nix-everything) + riscv64-unknown-linux-gnu + x86_64-unknown-freebsd + ; + }; + + inherit (hydraJobs) + manual + binaryTarball + binaryTarballCross + installerScript + installerScriptForGHA + rustInstaller + dockerImage + ; + + # Aggregate gating job: green ⇒ every artifact the upload script + # needs is available. `upload-release` can wait on this single job + # instead of the whole evaluation. Constituents are referenced by + # job *name* so that an evaluation failure in one of them does not + # take down the aggregate's own evaluation. + release = flake.inputs.nixpkgs.legacyPackages.x86_64-linux.releaseTools.aggregate { + name = "nix-release-${flake.packages.x86_64-linux.nix-everything.version}"; + meta.description = "Artifacts required for a Nix release"; + constituents = + let + collectJobNames = + prefix: x: + if lib.isDerivation x then + [ prefix ] + else if lib.isAttrs x then + lib.concatLists ( + lib.mapAttrsToList (n: collectJobNames (if prefix == "" then n else "${prefix}.${n}")) x + ) + else + [ ]; + in + collectJobNames "" (builtins.removeAttrs jobs [ "release" ]); + }; + }; +in +jobs diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix new file mode 100644 index 000000000000..98aec45e9e53 --- /dev/null +++ b/packaging/rust-installer/default.nix @@ -0,0 +1,88 @@ +# `NixOS/nix-installer` built with *this* Nix closure embedded, so +# Hydra/CI can dogfood the Rust installer without the (removed) +# `--nix-package-url` knob. +{ + lib, + stdenv, + buildPackages, + runCommand, + rustPlatform, + fetchFromGitHub, + tarball, +}: + +let + installerVersion = "2.34.6"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "nix-installer"; + tag = installerVersion; + hash = "sha256-aTaz8EtHexvke7tGr5MfeKy9g7AraIAFN+dPApm+fds="; + }; + + # Bare binary: no Nix closure yet. Appended below via `pack`, so the + # (expensive) Rust compile is independent of the embedded Nix and + # stays cacheable across Nix revisions. + bare = rustPlatform.buildRustPackage { + pname = "nix-installer-bare"; + version = installerVersion; + + inherit src; + + cargoHash = "sha256-/mNXkeZVuYsqd0TiUa7bzSP4xpKh0Fqga9EpasPbrzU="; + + doCheck = false; + + env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + ''; + }; +in + +runCommand "nix-installer-${tarball.passthru.nixVersion}" + { + nativeBuildInputs = [ + buildPackages.python3 + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + buildPackages.darwin.sigtool + buildPackages.darwin.cctools + ]; + + # The appended payload contains store-path strings on purpose; don't + # let the reference scanner pull the whole Nix closure into this + # derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + + passthru = { inherit bare; }; + + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; + } + '' + mkdir -p $out/bin $out/nix-support + + python3 ${src}/scripts/pack \ + --input ${bare}/bin/nix-installer \ + --tarball ${tarball}/nix.tar.zst \ + --nix-store-path ${tarball.passthru.nixStorePath} \ + --cacert-store-path ${tarball.passthru.cacertStorePath} \ + --nix-version ${tarball.passthru.nixVersion} \ + --output $out/bin/nix-installer + + install -m755 ${bare}/bin/nix-installer.sh $out/bin/nix-installer.sh + + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + '' diff --git a/packaging/rust-installer/tarball.nix b/packaging/rust-installer/tarball.nix new file mode 100644 index 000000000000..4f236e0215bb --- /dev/null +++ b/packaging/rust-installer/tarball.nix @@ -0,0 +1,50 @@ +# Zstd-compressed Nix closure in the layout expected by +# `NixOS/nix-installer` (`include_bytes!` at build time). +{ + lib, + stdenv, + runCommand, + buildPackages, + zstd, + nix, + cacert, +}: + +let + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ + nix + cacert + ]; + }; +in + +runCommand "nix-installer-tarball-${nix.version}" + { + nativeBuildInputs = [ zstd ]; + + passthru = { + nixStorePath = nix.outPath; + cacertStorePath = cacert.outPath; + nixVersion = nix.version; + }; + } + '' + mkdir -p $out + + dir=nix-${nix.version}-${stdenv.hostPlatform.system} + + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + + tar cf - \ + --sort=name \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) \ + | zstd -19 -T1 -o $out/nix.tar.zst + '' diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 7a61764d4f33..538a5e74d5ee 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -832,8 +832,8 @@ EOF # TODO: should probably alert the user if this is disabled? _sudo "to launch the Nix volume mounter" \ launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true - # TODO: confirm whether kickstart is necessesary? - # I feel a little superstitous, but it can guard + # TODO: confirm whether kickstart is necessary? + # I feel a little superstitious, but it can guard # against multiple problems (doesn't start, old # version still running for some reason...) _sudo "to launch the Nix volume mounter" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d4ea88b5ea6c..ae6625e1bc41 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -270,7 +270,7 @@ _diff() { printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" else - # simple colorized diff comatible w/ pre `--color` versions + # simple colorized diff compatible w/ pre `--color` versions diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" fi } @@ -961,7 +961,7 @@ configure_shell_profile() { cert_in_store() { # in a subshell # - change into the cert-file dir - # - get the phyiscal pwd + # - get the physical pwd # and test if this path is in the Nix store [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]] } diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index f17e4c2af3b9..73f389b6ff16 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -28,14 +28,15 @@ fi OS="$(uname -s)" -# macOS support for 10.12.6 or higher +# Since nixpkgs 25.11 the minimum deployment target is macOS 14.0 if [ "$OS" = "Darwin" ]; then + # shellcheck disable=SC2034 IFS='.' read -r macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF - if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + if [ "$macos_major" -lt 14 ]; then # patch may not be present; command substitution for simplicity - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 14.0 or higher" exit 1 fi fi diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 1a20dffd2459..93cb3c45a55b 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -53,7 +53,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index abf716cec6fc..201a56438950 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -58,7 +58,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build index 60cfd1514912..150e8aafcc3e 100644 --- a/src/clang-tidy-plugin/meson.build +++ b/src/clang-tidy-plugin/meson.build @@ -6,7 +6,7 @@ project( 'cpp_std=c++23', 'warning_level=2', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -20,6 +20,7 @@ llvm_dep = dependency('LLVM', version : '>= 16', required : true) sources = files( 'nix-clang-tidy-checks.cc', + 'nix-using-namespace.cc', ) # Build as a shared module (plugin) that can be loaded by clang-tidy --load diff --git a/src/clang-tidy-plugin/nix-clang-tidy-checks.cc b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc index 64ec63c2d319..bf69e7f1154f 100644 --- a/src/clang-tidy-plugin/nix-clang-tidy-checks.cc +++ b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc @@ -13,22 +13,19 @@ #include #include -namespace nix::clang_tidy { +#include "nix-using-namespace.hh" -using namespace clang; -using namespace clang::tidy; +namespace nix::clang_tidy { -class NixClangTidyChecks : public ClangTidyModule +class NixClangTidyChecks : public clang::tidy::ClangTidyModule { public: - void addCheckFactories([[maybe_unused]] ClangTidyCheckFactories & CheckFactories) override + void addCheckFactories([[maybe_unused]] clang::tidy::ClangTidyCheckFactories & CheckFactories) override { - // Custom checks will be registered here. - // Example: - // CheckFactories.registerCheck("nix-my-custom-check"); + CheckFactories.registerCheck("nix-using-namespace"); } }; -static ClangTidyModuleRegistry::Add X("nix-module", "Adds Nix-specific checks"); +static clang::tidy::ClangTidyModuleRegistry::Add X("nix-module", "Adds Nix-specific checks"); } // namespace nix::clang_tidy diff --git a/src/clang-tidy-plugin/nix-using-namespace.cc b/src/clang-tidy-plugin/nix-using-namespace.cc new file mode 100644 index 000000000000..69e1fdf73a84 --- /dev/null +++ b/src/clang-tidy-plugin/nix-using-namespace.cc @@ -0,0 +1,26 @@ +#include "nix-using-namespace.hh" + +#include +#include + +namespace nix::clang_tidy { + +void UsingNamespaceInNamespaceScopeCheck::registerMatchers(clang::ast_matchers::MatchFinder * Finder) +{ + Finder->addMatcher(clang::ast_matchers::usingDirectiveDecl().bind("usingNamespace"), this); +} + +void UsingNamespaceInNamespaceScopeCheck::check(const clang::ast_matchers::MatchFinder::MatchResult & Result) +{ + const auto * U = Result.Nodes.getNodeAs("usingNamespace"); + const clang::SourceLocation Loc = U->getBeginLoc(); + if (U->isImplicit() || !Loc.isValid() || U->getParentFunctionOrMethod()) + return; + + diag( + Loc, + "do not use using namespace directive in namespace scopes - keep those local to functions or explicitly qualify names." + "This is to reduce namespace pollution with unity builds."); +} + +} // namespace nix::clang_tidy diff --git a/src/clang-tidy-plugin/nix-using-namespace.hh b/src/clang-tidy-plugin/nix-using-namespace.hh new file mode 100644 index 000000000000..48cc3ebcb5f2 --- /dev/null +++ b/src/clang-tidy-plugin/nix-using-namespace.hh @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace nix::clang_tidy { + +/** + * Check that forbids instances on `using namespace ...;` in a namespace + * scope. + * + * This is because we rely on unity builds in certain situations (faster + * non-incremental builds, static initialiser issues), and the common pattern of + * doing `using namespace` in a translation unit is a big footgun. + * + * Based on `google-build-using-namespace`, modulo that `using namespace` in a + * non-namespace scope is fine (like in a function). + */ +class UsingNamespaceInNamespaceScopeCheck : public clang::tidy::ClangTidyCheck +{ +public: + UsingNamespaceInNamespaceScopeCheck(llvm::StringRef Name, clang::tidy::ClangTidyContext * Context) + : ClangTidyCheck(Name, Context) + { + } + + bool isLanguageVersionSupported(const clang::LangOptions & LangOpts) const override + { + return LangOpts.CPlusPlus; + } + + void registerMatchers(clang::ast_matchers::MatchFinder * Finder) override; + + void check(const clang::ast_matchers::MatchFinder::MatchResult & Result) override; +}; + +} // namespace nix::clang_tidy diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index 1903b36e589b..d96da3863e6f 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-external-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index 844cb262ee38..3976e4c3f81a 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-internal-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 8a0bde04b8ed..66dc9b758a44 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -6,7 +6,7 @@ project( 'nix-json-schema-checks', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 2c52678105f4..dd8d8a0f7e36 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,4 +1,5 @@ #include "nix/cmd/built-path.hh" +#include "nix/store/build-result.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" #include "nix/store/outputs-query.hh" @@ -124,4 +125,50 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const return res; } +SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { return SingleBuiltPath::Opaque{bo.path}; }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd{ + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built{ + .drvPath = make_ref(std::move(drvPath)), + .output = {bfd.output, outputPath}, + }; + }, + }, + b.raw()); +} + +BuiltPath toBuiltPath(KeyedBuildResult & result, ref evalStore, ref store) +{ + auto success = result.tryGetSuccess(); + assert(success); + return std::visit( + overloaded{ + [&](const DerivedPath::Built & bfd) { + std::map outputs; + for (auto & [outputName, realisation] : success->builtOutputs) + outputs.emplace(outputName, realisation.outPath); + BuiltPath bp = BuiltPath::Built{ + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }; + return bp; + }, + [&](const DerivedPath::Opaque & bo) { + BuiltPath bp = BuiltPath::Opaque{bo.path}; + return bp; + }, + }, + result.path.raw()); +} + } // namespace nix diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index d57b76411328..821c9e0565cd 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -440,9 +440,34 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu void MixOutLinkBase::createOutLinksMaybe(const std::vector & buildables, ref & store) { - if (outLink != "") + createOutLinksMaybe(toBuiltPaths(buildables), store); +} + +void MixOutLinkBase::createOutLinksMaybe(const BuiltPaths & paths, ref & store) +{ + if (outLink) if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, toBuiltPaths(buildables), *store2); + createOutLinks(*outLink, paths, *store2); +} + +void MixPrintOutPaths::printOutPathsMaybe(const BuiltPaths & paths, ref store) +{ + if (!printOutputPaths) + return; + + logger->stop(); + for (auto & path : paths) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { logger->cout(store->printStorePath(bo.path)); }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + logger->cout(store->printStorePath(output.second)); + } + }, + }, + path); + } } } // namespace nix diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 984bed34882e..f60c49d5e682 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -150,7 +150,7 @@ MixEvalArgs::MixEvalArgs() }); } -Bindings * MixEvalArgs::getAutoArgs(EvalState & state) +const Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); for (auto & [name, arg] : autoArgs) { diff --git a/src/libcmd/include/nix/cmd/built-path.hh b/src/libcmd/include/nix/cmd/built-path.hh index d41529e5ac4f..34761ee02dc8 100644 --- a/src/libcmd/include/nix/cmd/built-path.hh +++ b/src/libcmd/include/nix/cmd/built-path.hh @@ -105,4 +105,10 @@ struct BuiltPath : _BuiltPathRaw typedef std::vector BuiltPaths; +SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b); + +struct KeyedBuildResult; + +BuiltPath toBuiltPath(KeyedBuildResult & result, ref evalStore, ref store); + } // namespace nix diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index fc67a60b5b8a..92fd032de0cd 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -407,13 +407,16 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu struct MixOutLinkBase : virtual Args { /** Prefix for any output symlinks. Empty means do not write an output symlink. */ - std::filesystem::path outLink; + std::optional outLink = std::nullopt; - MixOutLinkBase(const std::string & defaultOutLink) + MixOutLinkBase(const std::optional & defaultOutLink) : outLink(defaultOutLink) { } + /** underlying function */ + void createOutLinksMaybe(const BuiltPaths & paths, ref & store); + /** smaller wrapper for convenience (historically this was the only one) */ void createOutLinksMaybe(const std::vector & buildables, ref & store); }; @@ -435,9 +438,25 @@ struct MixOutLinkByDefault : MixOutLinkBase, virtual Args addFlag({ .longName = "no-link", .description = "Do not create symlinks to the build results.", - .handler = {&outLink, std::filesystem::path{}}, + .handler = {[&] { outLink = std::nullopt; }}, }); } }; +struct MixPrintOutPaths : virtual Args +{ + bool printOutputPaths = false; + + MixPrintOutPaths() + { + addFlag({ + .longName = "print-out-paths", + .description = "Print the resulting output paths", + .handler = {&printOutputPaths, true}, + }); + } + + void printOutPathsMaybe(const BuiltPaths & paths, ref store); +}; + } // namespace nix diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 14897158ae6d..c265c834fb61 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -52,7 +52,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair MixEvalArgs(); - Bindings * getAutoArgs(EvalState & state); + const Bindings * getAutoArgs(EvalState & state); LookupPath lookupPath; diff --git a/src/libcmd/include/nix/cmd/markdown.hh b/src/libcmd/include/nix/cmd/markdown.hh index 95a59c2aa7b9..716b87b87eef 100644 --- a/src/libcmd/include/nix/cmd/markdown.hh +++ b/src/libcmd/include/nix/cmd/markdown.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include namespace nix { diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 7cba481059c9..9b9a03c3e423 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -1,8 +1,10 @@ #pragma once /// @file +#include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" +#include "nix/util/terminal.hh" #include "nix/util/types.hh" #include #include @@ -39,6 +41,8 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { std::filesystem::path historyFile; + bool isInteractive = nix::isTTY(getStandardInput()); + public: ReadlineLikeInteracter(std::filesystem::path historyFile) : historyFile(std::move(historyFile)) diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index 81c7b8df5a2d..5966a56a9574 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -9,7 +9,7 @@ namespace nix { struct AbstractNixRepl { ref state; - Bindings * autoArgs; + const Bindings * autoArgs; AbstractNixRepl(ref state) : state(state) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa79317..e01475104b5a 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -55,6 +55,13 @@ struct ServeUnixSocketOptions mode_t socketMode = 0666; #ifndef _WIN32 + /** + * Name of the socket for socket activation, as included in `LISTEN_FDNAMES` + * Ordinarily the name of the socket unit, e.g. `nix-daemon.socket` + * If this field is empty, no name filtering will be performed. + */ + std::string activationName = ""; + /** * Additional file descriptor to poll. Useful for doing a self-pipe trick * https://cr.yp.to/docs/selfpipe.html. @@ -68,6 +75,8 @@ struct ServeUnixSocketOptions #endif }; +MakeError(AbortServeSocket, BaseError); + /** * Run a server loop that accepts connections and calls the handler for each. * @@ -83,6 +92,7 @@ struct ServeUnixSocketOptions * * This function never returns normally. It runs until interrupted * (e.g., via SIGINT), at which point it throws `Interrupted`. + * Can be explicitly exited by throwing AbortServeSocket. * * @param options Configuration for the server. * @param handler Callback invoked for each accepted connection. diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index aa5da2e56646..d2d696c5cb8d 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -43,7 +43,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() return {*derivedPathWithInfo}; } - Bindings & autoArgs = *cmd.getAutoArgs(*state); + const Bindings & autoArgs = *cmd.getAutoArgs(*state); PackageInfos packageInfos; getDerivations(*state, *v, "", autoArgs, packageInfos, false); diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 1ef34a1ce014..5517f2d9741e 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -6,6 +6,7 @@ #include "nix/cmd/common-eval-args.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" +#include "nix/expr/eval-error.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" @@ -159,11 +160,16 @@ std::vector> InstallableFlake::getCursors(EvalState for (auto & attrPath : attrPaths) { debug("trying flake output attribute '%s'", attrPath); - auto attr = root->findAlongAttrPath(AttrPath::parse(state, attrPath)); - if (attr) { - res.push_back(ref(*attr)); - } else { - suggestions += attr.getSuggestions(); + try { + auto attr = root->findAlongAttrPath(AttrPath::parse(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); + } + } catch (TypeError & e) { + debug("error resolving attribute '%s': %s", attrPath, e.msg()); + // Continue to next attribute path } } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 3a167af3db49..92811c1d01da 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -54,8 +54,11 @@ InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std:: } else if (v.type() == nString) { + auto path = state->coerceToSingleDerivedPath(pos, v, errorCtx); + if (auto o = std::get_if(&path.raw())) + state->ensureLazyPathCopied(o->path); return {{ - .path = DerivedPath::fromSingle(state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .path = DerivedPath::fromSingle(path), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a25971ad7f89..6c0de1d8966a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,3 +1,4 @@ +#include "nix/cmd/built-path.hh" #include "nix/store/globals.hh" #include "nix/cmd/installables.hh" #include "nix/cmd/installable-derived-path.hh" @@ -14,6 +15,7 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" @@ -535,28 +537,6 @@ ref SourceExprCommand::parseInstallable(ref store, const std return installables.front(); } -static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { return SingleBuiltPath::Opaque{bo.path}; }, - [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { - auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); - // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. - SingleDerivedPath::Built truncatedBfd{ - .drvPath = makeConstantStorePathRef(drvPath.outPath()), - .output = bfd.output, - }; - auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); - return SingleBuiltPath::Built{ - .drvPath = make_ref(std::move(drvPath)), - .output = {bfd.output, outputPath}, - }; - }, - }, - b.raw()); -} - std::vector Installable::build( ref evalStore, ref store, Realise mode, const Installables & installables, BuildMode bMode) { @@ -652,36 +632,18 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + auto buildResults = store->getBuilder(evalStore)->buildPathsWithResults(pathsToBuild, bMode); throwBuildErrors(buildResults, *store); for (auto & buildResult : buildResults) { - // If we didn't throw, they must all be sucesses - auto & success = std::get(buildResult.inner); for (auto & aux : backmap[buildResult.path]) { - std::visit( - overloaded{ - [&](const DerivedPath::Built & bfd) { - std::map outputs; - for (auto & [outputName, realisation] : success.builtOutputs) - outputs.emplace(outputName, realisation.outPath); - res.push_back( - {aux.installable, - {.path = - BuiltPath::Built{ - .drvPath = - make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), - .outputs = outputs, - }, - .info = aux.info, - .result = buildResult}}); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back( - {aux.installable, - {.path = BuiltPath::Opaque{bo.path}, .info = aux.info, .result = buildResult}}); - }, + res.push_back({ + aux.installable, + BuiltPathWithResult{ + .path = toBuiltPath(buildResult, evalStore, store), + .info = aux.info, + .result = buildResult, }, - buildResult.path.raw()); + }); } } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index d970a8e4b066..9638b491f2c4 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 81240af7f547..5f2417dc1331 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -203,8 +203,25 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT setupSignals(); #endif - char * s = readline(promptForType(promptType)); - Finally doFree([&]() { free(s); }); + + /* Buffer for the non-interactive input. */ + std::string buffer; + const char * s = nullptr; + char * rl = nullptr; + + /* Use plain std::getline for non-interactive mode, which we also use for + testing purposes. readline/editline seem to disagree too much about how + to handle final prompts etc., so it's easier to bypass those. The tests + are mostly about testing the core repl logic, not input handling. */ + if (isInteractive) { + rl = ::readline(promptForType(promptType)); + s = rl; + } else { + s = std::getline(std::cin, buffer) ? buffer.c_str() : nullptr; + } + + Finally doFree([&]() { ::free(rl); }); + #ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); #endif @@ -215,15 +232,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - // editline doesn't echo the input to the output when non-interactive, unlike readline - // this results in a different behavior when running tests. The echoing is - // quite useful for reading the test output, so we add it here. + /* Echo the prompt into the output if run in non-interactive mode, somewhat + for the purposes of characterisation tests. */ if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { -#if !USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. std::cout << promptForType(promptType) << s << std::endl; -#endif } if (!s) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 777fc33256ca..ed35988368d8 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -26,6 +26,7 @@ #include "nix/util/finally.hh" #include "nix/cmd/markdown.hh" #include "nix/store/local-fs-store.hh" +#include "nix/store/build.hh" #include "nix/expr/print.hh" #include "nix/util/ref.hh" #include "nix/expr/value.hh" @@ -161,7 +162,24 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi return out; } -MakeError(IncompleteReplExpr, ParseError); +/** + * Thrown when the REPL's own input is incomplete (e.g. unclosed multi-line + * string or open parenthesis). The mainLoop catches this to prompt for + * continuation lines instead of showing an error. + * + * Only parseString and parseReplBindings may throw this. Evaluation can also + * produce "unexpected end of file" ParseErrors (e.g. `import ./broken.nix`), + * but those must be reported as errors, not trigger continuation. The + * exception subtype is what distinguishes the two cases. + */ +MakeError(IncompleteReplExpr, Error); + +static bool isIncompleteInput(const ParseError & e) +{ + return e.msg().find("unexpected end of file") != std::string::npos; +} + +void IncompleteReplExpr::anchor() {} static bool isFirstRepl = true; @@ -531,7 +549,7 @@ ProcessLineResult NixRepl::processLine(std::string line) std::string drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b" || command == ":bl") { - state->store->buildPaths({ + state->store->getBuilder()->buildPaths({ DerivedPath::Built{ .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}, @@ -660,11 +678,7 @@ ProcessLineResult NixRepl::processLine(std::string line) else { // Try parsing as bindings first (handles `x = 1`, `inherit ...`, etc.) - ExprAttrs * bindings = nullptr; - try { - bindings = parseReplBindings(line); - } catch (ParseError &) { - } + ExprAttrs * bindings = parseReplBindings(line); if (bindings) { Env * inheritEnv = bindings->inheritFromExprs ? bindings->buildInheritFromEnv(*state, *env) : nullptr; @@ -688,12 +702,13 @@ ProcessLineResult NixRepl::processLine(std::string line) void NixRepl::loadFile(const std::filesystem::path & path) { - loadedFiles.remove(path); - loadedFiles.push_back(path); Value v, v2; state->evalFile(lookupFileArg(*state, path.string()), v); state->autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); + // Remember for :reload only on success. + loadedFiles.remove(path); + loadedFiles.push_back(path); } void NixRepl::loadFlake(const std::string & flakeRefS) @@ -701,9 +716,6 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - loadedFlakes.remove(flakeRefS); - loadedFlakes.push_back(flakeRefS); - std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -730,6 +742,10 @@ void NixRepl::loadFlake(const std::string & flakeRefS) }), v); addAttrsToScope(v); + + // Remember for :reload only on success. + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); } void NixRepl::initEnv() @@ -772,28 +788,44 @@ void NixRepl::reloadFilesAndFlakes() void NixRepl::loadFiles() { - decltype(loadedFiles) old = loadedFiles; - loadedFiles.clear(); + // loadFile() rebuilds loadedFiles; keep failed entries and continue. + decltype(loadedFiles) old; + std::swap(old, loadedFiles); for (auto & i : old) { notice("Loading %1%...", PathFmt(i)); - loadFile(i); + try { + loadFile(i); + } catch (Error & e) { + loadedFiles.push_back(i); + printMsg(lvlError, e.msg()); + } } for (auto & [i, what] : getValues()) { notice("Loading installable '%1%'...", what); - addAttrsToScope(*i); + try { + addAttrsToScope(*i); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } } } void NixRepl::loadFlakes() { - Strings old = loadedFlakes; - loadedFlakes.clear(); + // See loadFiles(). + Strings old; + std::swap(old, loadedFlakes); for (auto & i : old) { notice("Loading flake '%1%'...", i); - loadFlake(i); + try { + loadFlake(i); + } catch (Error & e) { + loadedFlakes.push_back(i); + printMsg(lvlError, e.msg()); + } } } @@ -854,12 +886,9 @@ Expr * NixRepl::parseString(std::string s) try { return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } catch (ParseError & e) { - if (e.msg().find("unexpected end of file") != std::string::npos) - // For parse errors on incomplete input, we continue waiting for the next line of - // input without clearing the input so far. + if (isIncompleteInput(e)) throw IncompleteReplExpr(e.msg()); - else - throw; + throw; } } @@ -868,20 +897,20 @@ ExprAttrs * NixRepl::parseReplBindings(std::string s) auto basePath = state->rootPath("."); // Try parsing as bindings - std::exception_ptr bindingsError; try { return state->parseReplBindings(s, basePath, staticEnv); } catch (ParseError &) { - bindingsError = std::current_exception(); } // Try with semicolon appended (for `inherit foo` shorthand) // Use original source (s) for error messages, not s + ";" try { return state->parseReplBindings(s + ";", s, basePath, staticEnv); - } catch (ParseError &) { - // Semicolon retry failed; rethrow the original bindings error - std::rethrow_exception(bindingsError); + } catch (ParseError & e) { + if (isIncompleteInput(e)) + throw IncompleteReplExpr(e.msg()); + // Semicolon retry also failed; not valid binding syntax. + return nullptr; } } diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba462207..d5fbe0cc70aa 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -5,6 +5,7 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/util/signals.hh" +#include "nix/util/strings.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" @@ -20,6 +21,8 @@ namespace nix::unix { +void AbortServeSocket::anchor() {} + PeerInfo getPeerInfo(Descriptor remote) { PeerInfo peer; @@ -65,9 +68,16 @@ PeerInfo getPeerInfo(Descriptor remote) if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid())) throw Error("unexpected systemd environment variables"); + + auto fdNames = tokenizeString>(getEnv("LISTEN_FDNAMES").value_or(""), ":"); auto count = string2Int(*listenFds); assert(count); for (unsigned int i = 0; i < count; ++i) { + // Not all implementations of LISTEN_FDS will implement names, + // listen anyway if we do not have enough names + if (i < fdNames.size() && options.activationName != "" && fdNames[i] != options.activationName) + continue; + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); closeOnExec(fdSocket.get()); listeningSockets.push_back(std::move(fdSocket)); @@ -122,6 +132,9 @@ PeerInfo getPeerInfo(Descriptor remote) handler(std::move(remote), [&]() { listeningSockets.clear(); }); } + } catch (AbortServeSocket &) { + /* Explicitly aborted, bail out. */ + throw; } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index c47704ce4112..fb3a17ff2998 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -30,6 +30,7 @@ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/common') sources = files( + 'nix_api_eval.cc', 'nix_api_expr.cc', 'nix_api_external.cc', 'nix_api_value.cc', @@ -47,16 +48,33 @@ headers = files( subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixexprc', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'soversion' : nix_soversion, + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixexprc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixexprc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libexpr-c/meson.options b/src/libexpr-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libexpr-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libexpr-c/nix_api_eval.cc b/src/libexpr-c/nix_api_eval.cc new file mode 100644 index 000000000000..4c7e2e08fc35 --- /dev/null +++ b/src/libexpr-c/nix_api_eval.cc @@ -0,0 +1,55 @@ +#include "nix/expr/eval.hh" +#include "nix/expr/get-drvs.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +static const nix::Bindings & get_bindings_or_empty(nix::EvalState & state, nix_value * autoArgs) +{ + if (!autoArgs) { + return nix::Bindings::emptyBindings; + } + auto & v = check_value_in(autoArgs); + state.forceAttrs(v, nix::noPos, "while evaluating automatic function arguments"); + return *v.attrs(); +} + +extern "C" { + +StorePath * +nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value, bool ignoreAssertionFailures) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + auto maybePkg = nix::getDerivation(state->state, v, ignoreAssertionFailures); + if (!maybePkg) { + return nullptr; + } + nix::StorePath sp = maybePkg->requireDrvPath(); + return new StorePath{std::move(sp)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_value_auto_call_function( + nix_c_context * context, EvalState * state, nix_value * auto_args, nix_value * fn_val, nix_value * result) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & fn = check_value_in(fn_val); + auto & res = check_value_not_null(result); + + auto & b = get_bindings_or_empty(state->state, auto_args); + state->state.autoCallFunction(b, fn, res); + } + NIXC_CATCH_ERRS +} + +} // extern "C" diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 97680ac6bfe7..2387ee8c38c2 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,27 +19,6 @@ # include #endif -/** - * @brief Allocate and initialize using self-reference - * - * This allows a brace initializer to reference the object being constructed. - * - * @warning Use with care, as the pointer points to an object that is not fully constructed yet. - * - * @tparam T Type to allocate - * @tparam F A function type for `init`, taking a T* and returning the initializer for T - * @param init Function that takes a T* and returns the initializer for T - * @return Pointer to allocated and initialized object - */ -template -static T * unsafe_new_with_self(F && init) -{ - // Allocate - void * p = ::operator new(sizeof(T), static_cast(alignof(T))); - // Initialize with placement new - return new (p) T(init(static_cast(p))); -} - extern "C" { nix_err nix_libexpr_init(nix_c_context * context) @@ -129,23 +108,20 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - }); + auto readOnly = nix::make_ref(true); + return new nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ *readOnly}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = readOnly, + }; } NIXC_CATCH_ERRS_NULL } void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - if (builder) - builder->~nix_eval_state_builder(); - operator delete(builder, static_cast(alignof(nix_eval_state_builder))); + delete builder; } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -154,7 +130,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil context->last_err_code = NIX_OK; try { // TODO: load in one go? - builder->settings.readOnlyMode = nix::settings.readOnlyMode; + builder->settings.readOnlyMode = &nix::settings.readOnlyMode; loadConfFile(builder->settings); loadConfFile(builder->fetchSettings); } diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 3623ee076f6f..0326e7becf4a 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -342,6 +342,69 @@ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * o /** @} */ // doxygen group GC +/** @defgroup libexpr_eval Evaluation + * @ingroup libexpr + * @brief Higher-level evaluation helpers + * @{ + */ + +/** + * @brief Determine whether a Nix value is a derivation and, if so, return its + * store derivation path. + * + * Forces @p value and inspects it. The value is considered a derivation when it + * is an attribute set whose `type` attribute is the string `"derivation"`; in + * that case its `drvPath` attribute is parsed and returned. Otherwise NULL is + * returned without recording an error. + * + * Only the derivation path is returned. Other metadata (`name`, `system`, + * outputs, `meta`, ...) lives on @p value itself and can be read with the + * attribute-set accessors such as nix_get_attr_byname() and nix_get_string(). + * + * @param[out] context Optional, stores error information. On a NULL return, + * inspect the error code via nix_err_code() to tell the two NULL cases apart: + * NIX_OK means @p value is simply not a derivation, any other code means + * inspection failed. See @ref errors. + * @param[in] state The evaluation state. + * @param[in] value The value to inspect. It is forced by this call. + * @param[in] ignoreAssertionFailures If true, an assertion failure raised while + * forcing @p value is treated as "not a derivation" (NULL is returned without + * an error) rather than being reported as an error. + * @return A newly allocated StorePath holding the derivation path, or NULL. + * Free a non-NULL result with nix_store_path_free(). + */ +StorePath * +nix_get_derivation(nix_c_context * context, EvalState * state, nix_value * value, bool ignoreAssertionFailures); + +/** + * @brief Call a function, drawing its arguments from an attribute set. + * + * Forces @p fn_val and writes the application result into @p result. The result + * is not forced; call nix_value_force() to evaluate it before inspecting the + * final value. + * + * - If @p fn_val is a function that takes a set of named arguments + * (e.g. `{ a, b ? 1 }: ...`), it is called with an attribute set assembled + * from @p auto_args: each named argument is taken from @p auto_args when + * present; an argument absent from @p auto_args falls back to its default; + * an argument that is both absent and has no default is an error. + * - Otherwise @p fn_val is copied into @p result unchanged. This includes any + * non-function value as well as a function that takes a single unnamed + * argument (e.g. `x: ...`), since there are no named arguments to supply. + * + * @param[out] context Optional, stores error information + * @param[in] state The evaluation state. + * @param[in] auto_args Attribute set value supplying the named arguments, or + * NULL to supply none. + * @param[in] fn_val The value to call. + * @param[out] result Pre-allocated nix_value that receives the result. + * @return NIX_OK if the call was successful, an error code otherwise. + */ +nix_err nix_value_auto_call_function( + nix_c_context * context, EvalState * state, nix_value * auto_args, nix_value * fn_val, nix_value * result); + +/** @} */ // doxygen group libexpr_eval + // cffi end #ifdef __cplusplus } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b38aeaf7b498..16b32bfa4602 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -2,6 +2,7 @@ #define NIX_API_EXPR_INTERNAL_H #include +#include #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval.hh" @@ -18,8 +19,7 @@ struct nix_eval_state_builder nix::EvalSettings settings; nix::fetchers::Settings fetchSettings; nix::LookupPath lookupPath; - // TODO: make an EvalSettings setting own this instead? - bool readOnlyMode; + nix::ref readOnlyMode; }; struct EvalState @@ -75,4 +75,35 @@ struct nix_realised_string } // extern "C" +// Shared helpers for validating nix_value [in] parameters across libexpr-c translation units. +inline const nix::Value & check_value_not_null(const nix_value * value) +{ + if (!value || !value->value) + throw std::runtime_error("nix_value is null"); + return *value->value; +} + +inline nix::Value & check_value_not_null(nix_value * value) +{ + if (!value || !value->value) + throw std::runtime_error("nix_value is null"); + return *value->value; +} + +inline const nix::Value & check_value_in(const nix_value * value) +{ + auto & v = check_value_not_null(value); + if (!v.isValid()) + throw std::runtime_error("Uninitialized nix_value"); + return v; +} + +inline nix::Value & check_value_in(nix_value * value) +{ + auto & v = check_value_not_null(value); + if (!v.isValid()) + throw std::runtime_error("Uninitialized nix_value"); + return v; +} + #endif // NIX_API_EXPR_INTERNAL_H diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 98b68d9f6b49..cf2fa2117273 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -41,6 +41,8 @@ nix_err nix_external_add_string_context(nix_c_context * context, nix_string_cont } // extern "C" +namespace { + class NixCExternalValue : public nix::ExternalValueBase { NixCExternalValueDesc & desc; @@ -166,6 +168,8 @@ class NixCExternalValue : public nix::ExternalValueBase virtual ~NixCExternalValue() override {}; }; +} // namespace + extern "C" { ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 3cb5705898c4..eaa5c2e1d535 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -12,41 +12,6 @@ #include "nix_api_store_internal.h" #include "nix_api_value.h" -// Internal helper functions to check [in] and [out] `Value *` parameters -static const nix::Value & check_value_not_null(const nix_value * value) -{ - if (!value) { - throw std::runtime_error("nix_value is null"); - } - return *value->value; -} - -static nix::Value & check_value_not_null(nix_value * value) -{ - if (!value) { - throw std::runtime_error("nix_value is null"); - } - return *value->value; -} - -static const nix::Value & check_value_in(const nix_value * value) -{ - auto & v = check_value_not_null(value); - if (!v.isValid()) { - throw std::runtime_error("Uninitialized nix_value"); - } - return v; -} - -static nix::Value & check_value_in(nix_value * value) -{ - auto & v = check_value_not_null(value); - if (!v.isValid()) { - throw std::runtime_error("Uninitialized nix_value"); - } - return v; -} - static nix::Value & check_value_out(nix_value * value) { auto & v = check_value_not_null(value); @@ -335,7 +300,7 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value) if (context) context->last_err_code = NIX_OK; try { - auto & v = check_value_out(value); + auto & v = check_value_in(value); assert(v.type() == nix::nExternal); return (ExternalValue *) v.external(); } diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 694fbc1fe789..9a7a46c1663d 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -8,6 +8,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -25,7 +26,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -37,6 +38,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index df28661b7e78..4a87bb4545fd 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index ca7996acc16e..2483b3ea811b 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,15 +1,13 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/expr/tests/value/context.hh" namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::map(gen::arbitrary(), [](StorePath drvPath) { return NixStringContextElem::DrvDeep{ .drvPath = drvPath, @@ -17,8 +15,9 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat( gen::inRange(0, std::variant_size_v), [](uint8_t n) -> Gen { diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index c685f6a094a8..3997280ee79e 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,8 +1,6 @@ #include #include -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" @@ -20,8 +18,6 @@ class DerivedPathExpressionTest : public LibExprTest // See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args TEST_F(DerivedPathExpressionTest, force_init) {} -#ifndef COVERAGE - RC_GTEST_FIXTURE_PROP(DerivedPathExpressionTest, prop_opaque_path_round_trip, (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); @@ -63,6 +59,4 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(SingleDerivedPath{b} == d); } -#endif - } /* namespace nix */ diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index e722cc48499a..918ea71807e2 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -5,14 +5,14 @@ namespace nix { -using namespace testing; - // Testing eval of PrimOp's class ErrorTraceTest : public LibExprTest {}; TEST_F(ErrorTraceTest, TraceBuilder) { + using namespace testing; + ASSERT_THROW(state.error("puppy").debugThrow(), EvalError); ASSERT_THROW(state.error("puppy").withTrace(noPos, "doggy").debugThrow(), EvalError); @@ -54,1270 +54,4 @@ TEST_F(ErrorTraceTest, NestedThrows) } } -#define ASSERT_TRACE1(args, type, message) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE2(args, type, message, context) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE3(args, type, message, context1, context2) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context3)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -// We assume that expr starts with "builtins.derivationStrict { name =", -// otherwise the name attribute position (1, 29) would be invalid. -#define DERIVATION_TRACE_HINTFMT(name) \ - HintFmt( \ - "while evaluating derivation '%s'\n" \ - " whose name attribute is located at %s", \ - name, \ - Pos(1, 29, Pos::String{.source = make_ref(expr)})) - -// To keep things simple, we also assume that derivation name is "foo". -#define ASSERT_DERIVATION_TRACE1(args, type, message) \ - ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ - ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ - ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) - -TEST_F(ErrorTraceTest, replaceStrings) -{ - ASSERT_TRACE2( - "replaceStrings 0 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [] 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); - - ASSERT_TRACE1( - "replaceStrings [ 0 ] [] {}", - EvalError, - HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); - - ASSERT_TRACE2( - "replaceStrings [ 1 ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"oo\" ] [ true ] \"foo\"", - TypeError, - HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"old\" ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); -} - -TEST_F(ErrorTraceTest, scopedImport) {} - -TEST_F(ErrorTraceTest, import) {} - -TEST_F(ErrorTraceTest, typeOf) {} - -TEST_F(ErrorTraceTest, isNull) {} - -TEST_F(ErrorTraceTest, isFunction) {} - -TEST_F(ErrorTraceTest, isInt) {} - -TEST_F(ErrorTraceTest, isFloat) {} - -TEST_F(ErrorTraceTest, isString) {} - -TEST_F(ErrorTraceTest, isBool) {} - -TEST_F(ErrorTraceTest, isPath) {} - -TEST_F(ErrorTraceTest, break) {} - -TEST_F(ErrorTraceTest, abort) {} - -TEST_F(ErrorTraceTest, throw) {} - -TEST_F(ErrorTraceTest, addErrorContext) {} - -TEST_F(ErrorTraceTest, ceil) -{ - ASSERT_TRACE2( - "ceil \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.ceil")); -} - -TEST_F(ErrorTraceTest, floor) -{ - ASSERT_TRACE2( - "floor \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.floor")); -} - -TEST_F(ErrorTraceTest, tryEval) {} - -TEST_F(ErrorTraceTest, getEnv) -{ - ASSERT_TRACE2( - "getEnv [ ]", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getEnv")); -} - -TEST_F(ErrorTraceTest, seq) {} - -TEST_F(ErrorTraceTest, deepSeq) {} - -TEST_F(ErrorTraceTest, trace) {} - -TEST_F(ErrorTraceTest, placeholder) -{ - ASSERT_TRACE2( - "placeholder []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.placeholder")); -} - -TEST_F(ErrorTraceTest, toPath) -{ - ASSERT_TRACE2( - "toPath []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.toPath")); - - ASSERT_TRACE2( - "toPath \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the first argument passed to builtins.toPath")); -} - -TEST_F(ErrorTraceTest, storePath) -{ - ASSERT_TRACE2( - "storePath true", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); -} - -TEST_F(ErrorTraceTest, pathExists) -{ - ASSERT_TRACE2( - "pathExists []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while realising the context of a path")); - - ASSERT_TRACE2( - "pathExists \"zorglub\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), - HintFmt("while realising the context of a path")); -} - -TEST_F(ErrorTraceTest, baseNameOf) -{ - ASSERT_TRACE2( - "baseNameOf []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); -} - -TEST_F(ErrorTraceTest, dirOf) {} - -TEST_F(ErrorTraceTest, readFile) {} - -TEST_F(ErrorTraceTest, findFile) {} - -TEST_F(ErrorTraceTest, hashFile) {} - -TEST_F(ErrorTraceTest, readDir) {} - -TEST_F(ErrorTraceTest, toXML) {} - -TEST_F(ErrorTraceTest, toJSON) {} - -TEST_F(ErrorTraceTest, fromJSON) {} - -TEST_F(ErrorTraceTest, toFile) {} - -TEST_F(ErrorTraceTest, filterSource) -{ - ASSERT_TRACE2( - "filterSource [] []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] ./.", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.filterSource")); - - // Unsupported by store "dummy" - - // ASSERT_TRACE2("filterSource (_: 1) ./.", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while adding path '/home/layus/projects/nix'")); - - // ASSERT_TRACE2("filterSource (_: _: 1) ./.", - // TypeError, - // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), - // HintFmt("while evaluating the return value of the path filter function")); -} - -TEST_F(ErrorTraceTest, path) {} - -TEST_F(ErrorTraceTest, attrNames) -{ - ASSERT_TRACE2( - "attrNames []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrNames")); -} - -TEST_F(ErrorTraceTest, attrValues) -{ - ASSERT_TRACE2( - "attrValues []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrValues")); -} - -TEST_F(ErrorTraceTest, getAttr) -{ - ASSERT_TRACE2( - "getAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" {}", - TypeError, - HintFmt("attribute '%s' missing", "foo"), - HintFmt("in the attribute set under consideration")); -} - -TEST_F(ErrorTraceTest, unsafeGetAttrPos) {} - -TEST_F(ErrorTraceTest, hasAttr) -{ - ASSERT_TRACE2( - "hasAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.hasAttr")); - - ASSERT_TRACE2( - "hasAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.hasAttr")); -} - -TEST_F(ErrorTraceTest, isAttrs) {} - -TEST_F(ErrorTraceTest, removeAttrs) -{ - ASSERT_TRACE2( - "removeAttrs \"\" \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ \"1\" ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); -} - -TEST_F(ErrorTraceTest, listToAttrs) -{ - ASSERT_TRACE2( - "listToAttrs 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the argument passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ {} ]", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in a {name=...; value=...;} pair")); - - ASSERT_TRACE2( - "listToAttrs [ { name = 1; } ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ { name = \"foo\"; } ]", - TypeError, - HintFmt("attribute '%s' missing", "value"), - HintFmt("in a {name=...; value=...;} pair")); -} - -TEST_F(ErrorTraceTest, intersectAttrs) -{ - ASSERT_TRACE2( - "intersectAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); - - ASSERT_TRACE2( - "intersectAttrs {} []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); -} - -TEST_F(ErrorTraceTest, catAttrs) -{ - ASSERT_TRACE2( - "catAttrs [] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); -} - -TEST_F(ErrorTraceTest, functionArgs) -{ - ASSERT_TRACE1("functionArgs {}", TypeError, HintFmt("'functionArgs' requires a function")); -} - -TEST_F(ErrorTraceTest, mapAttrs) -{ - ASSERT_TRACE2( - "mapAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); - - // XXX: deferred - // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", - // TypeError, - // HintFmt("cannot coerce %s to a string", "an integer"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, zipAttrsWith) -{ - ASSERT_TRACE2( - "zipAttrsWith [] [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); - - ASSERT_TRACE2( - "zipAttrsWith (_: 1) [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); - - // XXX: How to properly tell that the function takes two arguments ? - // The same question also applies to sort, and maybe others. - // Due to laziness, we only create a thunk, and it fails later on. - // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while evaluating the attribute 'foo'")); - - // XXX: Also deferred deeply - // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", - // TypeError, - // HintFmt("cannot coerce %s to a string", "a list"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, isList) {} - -TEST_F(ErrorTraceTest, elemAt) -{ - ASSERT_TRACE2( - "elemAt \"foo\" (-1)", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); - - ASSERT_TRACE1( - "elemAt [] (-1)", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); - - ASSERT_TRACE1( - "elemAt [\"foo\"] 3", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); -} - -TEST_F(ErrorTraceTest, head) -{ - ASSERT_TRACE2( - "head 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.head'")); - - ASSERT_TRACE1("head []", Error, HintFmt("'builtins.head' called on an empty list")); -} - -TEST_F(ErrorTraceTest, tail) -{ - ASSERT_TRACE2( - "tail 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.tail'")); - - ASSERT_TRACE1("tail []", Error, HintFmt("'builtins.tail' called on an empty list")); -} - -TEST_F(ErrorTraceTest, map) -{ - ASSERT_TRACE2( - "map 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.map")); - - ASSERT_TRACE2( - "map 1 [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.map")); -} - -TEST_F(ErrorTraceTest, filter) -{ - ASSERT_TRACE2( - "filter 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter (_: 5) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); -} - -TEST_F(ErrorTraceTest, elem) -{ - ASSERT_TRACE2( - "elem 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.elem")); -} - -TEST_F(ErrorTraceTest, concatLists) -{ - ASSERT_TRACE2( - "concatLists 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ 1 ]", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ [1] \"foo\" ]", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); -} - -TEST_F(ErrorTraceTest, length) -{ - ASSERT_TRACE2( - "length 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); - - ASSERT_TRACE2( - "length \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); -} - -TEST_F(ErrorTraceTest, foldlPrime) -{ - ASSERT_TRACE2( - "foldl' 1 \"foo\" true", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); - - ASSERT_TRACE2( - "foldl' (_: 1) \"foo\" true", - TypeError, - HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); - - ASSERT_TRACE1( - "foldl' (_: 1) \"foo\" [ true ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "foldl' (a: b: a && b) \"foo\" [ true ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("in the left operand of the AND (&&) operator")); -} - -TEST_F(ErrorTraceTest, any) -{ - ASSERT_TRACE2( - "any 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.any")); -} - -TEST_F(ErrorTraceTest, all) -{ - ASSERT_TRACE2( - "all 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.all")); -} - -TEST_F(ErrorTraceTest, genList) -{ - ASSERT_TRACE2( - "genList 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.genList")); - - ASSERT_TRACE2( - "genList 1 2", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.genList")); - - // XXX: deferred - // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", - // TypeError, - // HintFmt("cannot add %s to an integer", "a string"), - // HintFmt("while evaluating anonymous lambda")); - - ASSERT_TRACE1("genList false (-3)", EvalError, HintFmt("cannot create list of size %d", -3)); -} - -TEST_F(ErrorTraceTest, sort) -{ - ASSERT_TRACE2( - "sort 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.sort")); - - ASSERT_TRACE2( - "sort 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.sort")); - - ASSERT_TRACE1( - "sort (_: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "sort (_: _: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); - - // XXX: Trace too deep, need better asserts - // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s", "a string", "a set")); - - // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); -} - -TEST_F(ErrorTraceTest, partition) -{ - ASSERT_TRACE2( - "partition 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); -} - -TEST_F(ErrorTraceTest, groupBy) -{ - ASSERT_TRACE2( - "groupBy 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (x: x) [ \"foo\" \"bar\" 1 ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); -} - -TEST_F(ErrorTraceTest, concatMap) -{ - ASSERT_TRACE2( - "concatMap 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) [ \"foo\" ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: \"foo\") [ 1 2 ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); -} - -TEST_F(ErrorTraceTest, add) -{ - ASSERT_TRACE2( - "add \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the addition")); - - ASSERT_TRACE2( - "add 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the addition")); -} - -TEST_F(ErrorTraceTest, sub) -{ - ASSERT_TRACE2( - "sub \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the subtraction")); - - ASSERT_TRACE2( - "sub 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the subtraction")); -} - -TEST_F(ErrorTraceTest, mul) -{ - ASSERT_TRACE2( - "mul \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the multiplication")); - - ASSERT_TRACE2( - "mul 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the multiplication")); -} - -TEST_F(ErrorTraceTest, div) -{ - ASSERT_TRACE2( - "div \"foo\" 1 # TODO: an integer was expected -> a number", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first operand of the division")); - - ASSERT_TRACE2( - "div 1 \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second operand of the division")); - - ASSERT_TRACE1("div \"foo\" 0", EvalError, HintFmt("division by zero")); -} - -TEST_F(ErrorTraceTest, bitAnd) -{ - ASSERT_TRACE2( - "bitAnd 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitAnd")); - - ASSERT_TRACE2( - "bitAnd 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitAnd")); -} - -TEST_F(ErrorTraceTest, bitOr) -{ - ASSERT_TRACE2( - "bitOr 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitOr")); - - ASSERT_TRACE2( - "bitOr 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitOr")); -} - -TEST_F(ErrorTraceTest, bitXor) -{ - ASSERT_TRACE2( - "bitXor 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitXor")); - - ASSERT_TRACE2( - "bitXor 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitXor")); -} - -TEST_F(ErrorTraceTest, lessThan) -{ - ASSERT_TRACE1( - "lessThan 1 \"foo\"", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL))); - - ASSERT_TRACE1( - "lessThan {} {}", - EvalError, - HintFmt( - "cannot compare %s with %s; values of that type are incomparable (values are %s and %s)", - "a set", - "a set", - Uncolored("{ }"), - Uncolored("{ }"))); - - ASSERT_TRACE2( - "lessThan [ 1 2 ] [ \"foo\" ]", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while comparing two list elements")); -} - -TEST_F(ErrorTraceTest, toString) -{ - ASSERT_TRACE2( - "toString { a = 1; }", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), - HintFmt("while evaluating the first argument passed to builtins.toString")); -} - -TEST_F(ErrorTraceTest, substring) -{ - ASSERT_TRACE2( - "substring {} \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 3 \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 0 3 {}", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); - - ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, HintFmt("negative start position in 'substring'")); -} - -TEST_F(ErrorTraceTest, stringLength) -{ - ASSERT_TRACE2( - "stringLength {} # TODO: context is missing ???", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the argument passed to builtins.stringLength")); -} - -TEST_F(ErrorTraceTest, hashString) -{ - ASSERT_TRACE2( - "hashString 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.hashString")); - - ASSERT_TRACE1( - "hashString \"foo\" \"content\"", - UsageError, - HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); - - ASSERT_TRACE2( - "hashString \"sha256\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.hashString")); -} - -TEST_F(ErrorTraceTest, match) -{ - ASSERT_TRACE2( - "match 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.match")); - - ASSERT_TRACE2( - "match \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.match")); - - ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, HintFmt("invalid regular expression '%s'", "(.*")); -} - -TEST_F(ErrorTraceTest, split) -{ - ASSERT_TRACE2( - "split 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.split")); - - ASSERT_TRACE2( - "split \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.split")); - - ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, HintFmt("invalid regular expression '%s'", "f(o*o")); -} - -TEST_F(ErrorTraceTest, concatStringsSep) -{ - ASSERT_TRACE2( - "concatStringsSep 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt( - "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); -} - -TEST_F(ErrorTraceTest, parseDrvName) -{ - ASSERT_TRACE2( - "parseDrvName 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); -} - -TEST_F(ErrorTraceTest, compareVersions) -{ - ASSERT_TRACE2( - "compareVersions 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.compareVersions")); - - ASSERT_TRACE2( - "compareVersions \"abd\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.compareVersions")); -} - -TEST_F(ErrorTraceTest, splitVersion) -{ - ASSERT_TRACE2( - "splitVersion 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.splitVersion")); -} - -TEST_F(ErrorTraceTest, traceVerbose) {} - -TEST_F(ErrorTraceTest, derivationStrict) -{ - ASSERT_TRACE2( - "derivationStrict \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", "\"\""), - HintFmt("while evaluating the argument passed to builtins.derivationStrict")); - - ASSERT_TRACE2( - "derivationStrict {}", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in the attrset passed as argument to builtins.derivationStrict")); - - ASSERT_TRACE3( - "derivationStrict { name = 1; }", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), - HintFmt("while evaluating the derivation attribute 'name'")); - - ASSERT_DERIVATION_TRACE1( - "derivationStrict { name = \"foo\"; }", EvalError, HintFmt("required attribute 'builder' missing")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", - EvalError, - HintFmt("expected a list but found %s: %s", "a string", "\"out\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", - EvalError, - HintFmt("derivation cannot have an empty set of outputs"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", - EvalError, - HintFmt("duplicate derivation output '%s'", "out"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); -} - } /* namespace nix */ diff --git a/src/libexpr-tests/lazy-fetcher-attr.cc b/src/libexpr-tests/lazy-fetcher-attr.cc new file mode 100644 index 000000000000..4c36424ecb55 --- /dev/null +++ b/src/libexpr-tests/lazy-fetcher-attr.cc @@ -0,0 +1,100 @@ +#include + +#include "nix/expr/fetch-tree.hh" +#include "nix/expr/tests/libexpr.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/store/path.hh" + +namespace nix { + +class LazyFetcherAttrTest : public LibExprTest +{ +protected: + StorePath dummyPath() + { + return StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-test"}; + } +}; + +TEST_F(LazyFetcherAttrTest, nonLazyAttrProducesImmediateValue) +{ + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("revCount", uint64_t(5)); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 5); +} + +TEST_F(LazyFetcherAttrTest, lazyAttrProducesThunk) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(42); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + + // Not yet forced, so the lazy function should not have been called + EXPECT_EQ(calls, 0); + + // Force the thunk + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 42); + EXPECT_EQ(calls, 1); +} + +TEST_F(LazyFetcherAttrTest, lazyFunctionOnlyCalledOnAccess) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("lastModified", uint64_t(1000)); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(99); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + // Access lastModified, so should not trigger lazy revCount + auto * lmAttr = v.attrs()->get(state.symbols.create("lastModified")); + ASSERT_NE(lmAttr, nullptr); + state.forceValue(*lmAttr->value, noPos); + EXPECT_EQ(lmAttr->value->integer().value, 1000); + EXPECT_EQ(calls, 0); + + // Now access revCount + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 99); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c20654..0927ee24b2b2 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -51,7 +51,9 @@ sources = files( 'error_traces.cc', 'eval.cc', 'json.cc', + 'lazy-fetcher-attr.cc', 'main.cc', + 'nix_api_eval.cc', 'nix_api_expr.cc', 'nix_api_external.cc', 'nix_api_value.cc', diff --git a/src/libexpr-tests/nix_api_eval.cc b/src/libexpr-tests/nix_api_eval.cc new file mode 100644 index 000000000000..9d2117678735 --- /dev/null +++ b/src/libexpr-tests/nix_api_eval.cc @@ -0,0 +1,259 @@ +#include "nix_api_store.h" +#include "nix_api_util.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" + +#include "nix/expr/tests/nix_api_expr.hh" +#include "nix/util/tests/string_callback.hh" +#include "nix/util/tests/gmock-matchers.hh" + +#include +#include + +namespace nixC { + +// nix_get_derivation + +TEST_F(nix_api_expr_test, nix_get_derivation_returns_drv_path) +{ + auto expr = R"(derivation { name = "myname"; builder = "mybuilder"; system = "mysystem"; })"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + assert_ctx_ok(); + ASSERT_NE(nullptr, drvPath); + + std::string name; + nix_store_path_name(drvPath, OBSERVE_STRING(name)); + EXPECT_THAT(name, ::testing::HasSubstr("myname")); + EXPECT_THAT(name, ::testing::EndsWith(".drv")); + + nix_store_path_free(drvPath); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_non_derivation_returns_null_without_error) +{ + nix_expr_eval_from_string(ctx, state, "{ a = 1; }", ".", value); + assert_ctx_ok(); + + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + // Not a derivation: NULL with no error recorded, so the caller can tell + // this apart from a genuine failure. + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_non_attrset_returns_null_without_error) +{ + nix_expr_eval_from_string(ctx, state, "42", ".", value); + assert_ctx_ok(); + + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_null_value_is_error) +{ + StorePath * drvPath = nix_get_derivation(ctx, state, nullptr, false); + ASSERT_EQ(nullptr, drvPath); + ASSERT_NE(NIX_OK, nix_err_code(ctx)); +} + +// A derivation-shaped attribute set whose `name` throws an assertion only when +// forced. The outer set is already WHNF, so nix_get_derivation is what triggers +// the failure (while reading the name), exercising the assertion handling. +static constexpr const char * ASSERTING_DRV = R"({ type = "derivation"; name = assert false; "myname"; })"; + +TEST_F(nix_api_expr_test, nix_get_derivation_assertion_ignored) +{ + nix_expr_eval_from_string(ctx, state, ASSERTING_DRV, ".", value); + assert_ctx_ok(); + + // With ignoreAssertionFailures = true the assertion is swallowed and the + // value is reported as "not a derivation": NULL with no error. + StorePath * drvPath = nix_get_derivation(ctx, state, value, true); + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); +} + +TEST_F(nix_api_expr_test, nix_get_derivation_assertion_propagated) +{ + nix_expr_eval_from_string(ctx, state, ASSERTING_DRV, ".", value); + assert_ctx_ok(); + + // With ignoreAssertionFailures = false the assertion surfaces as an error. + StorePath * drvPath = nix_get_derivation(ctx, state, value, false); + ASSERT_EQ(nullptr, drvPath); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), ::testing::HasSubstr("assert")); +} + +// nix_value_auto_call_function + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_supplies_args) +{ + nix_expr_eval_from_string(ctx, state, "{ a, b }: a + b", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 1; b = 2; }", ".", args); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + assert_ctx_ok(); + + ASSERT_EQ(3, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_uses_defaults) +{ + nix_expr_eval_from_string(ctx, state, "{ a, b ? 10 }: a + b", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 5; }", ".", args); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + assert_ctx_ok(); + + ASSERT_EQ(15, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_forces_auto_args) +{ + nix_expr_eval_from_string(ctx, state, "{ a }: a + 1", ".", value); + assert_ctx_ok(); + + nix_value * identity = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "x: x", ".", identity); + assert_ctx_ok(); + + nix_value * attrs = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 4; }", ".", attrs); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_init_apply(ctx, args, identity, attrs); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + assert_ctx_ok(); + + ASSERT_EQ(5, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, identity); + nix_gc_decref(ctx, attrs); + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_non_attr_auto_args_is_error) +{ + nix_expr_eval_from_string(ctx, state, "{ a ? 7 }: a", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_init_int(ctx, args, 42); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, args, value, result); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), ::nix::testing::HasSubstrIgnoreANSIMatcher("expected a set")); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_null_args_uses_defaults) +{ + nix_expr_eval_from_string(ctx, state, "{ a ? 7 }: a", ".", value); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // NULL auto_args supplies no arguments; every formal must then have a + // default. + nix_value_auto_call_function(ctx, state, nullptr, value, result); + assert_ctx_ok(); + + ASSERT_EQ(7, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_missing_arg_is_error) +{ + nix_expr_eval_from_string(ctx, state, "{ a, b }: a + b", ".", value); + assert_ctx_ok(); + + nix_value * args = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "{ a = 1; }", ".", args); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // 'b' has neither a supplied value nor a default. The argument name in the + // message is colorized, so use the ANSI-stripping matcher to assert on it. + nix_value_auto_call_function(ctx, state, args, value, result); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), + ::nix::testing::HasSubstrIgnoreANSIMatcher( + "cannot evaluate a function that has an argument without a value ('b')")); + + nix_gc_decref(ctx, args); + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_non_function_passthrough) +{ + nix_expr_eval_from_string(ctx, state, "42", ".", value); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // A non-function value is returned unchanged. + nix_value_auto_call_function(ctx, state, nullptr, value, result); + assert_ctx_ok(); + + ASSERT_EQ(42, nix_get_int(ctx, result)); + + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_single_arg_lambda_passthrough) +{ + nix_expr_eval_from_string(ctx, state, "x: x + 1", ".", value); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + // A function taking a single unnamed argument has no named arguments to + // fill, so it is returned unchanged rather than being called. + nix_value_auto_call_function(ctx, state, nullptr, value, result); + assert_ctx_ok(); + + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, result)); + + nix_gc_decref(ctx, result); +} + +TEST_F(nix_api_expr_test, nix_value_auto_call_function_null_fn_is_error) +{ + nix_value * result = nix_alloc_value(ctx, state); + nix_value_auto_call_function(ctx, state, nullptr, nullptr, result); + ASSERT_NE(NIX_OK, nix_err_code(ctx)); + + nix_gc_decref(ctx, result); +} + +} // namespace nixC diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index c3a3f2dd53b1..8362e6850892 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -20,8 +20,8 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) auto delTmpDir = std::make_unique(tmpDir, true); auto nixpkgs = tmpDir / "pkgs"; auto nixos = tmpDir / "cfg"; - std::filesystem::create_directories(nixpkgs); - std::filesystem::create_directories(nixos); + nix::createDirs(nixpkgs); + nix::createDirs(nixos); std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string(); std::string nixosEntry = "nixos-config=" + nixos.string(); diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index e17a52c31dfc..408fbaa83d73 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -104,4 +104,34 @@ TEST_F(nix_api_expr_test, nix_external_printValueAsJSON_can_use_state) nix_gc_decref(ctx, toJsonFn); } +TEST_F(nix_api_expr_test, nix_get_external_roundtrip) +{ + int content = 42; + NixCExternalValueDesc desc{}; + ExternalValue * ext = nix_create_external_value(ctx, &desc, &content); + assert_ctx_ok(); + ASSERT_NE(nullptr, ext); + + nix_init_external(ctx, value, ext); + assert_ctx_ok(); + + nix_value_force(ctx, state, value); + assert_ctx_ok(); + + ExternalValue * retrieved = nix_get_external(ctx, value); + assert_ctx_ok(); + ASSERT_NE(nullptr, retrieved); + + void * content_ptr = nix_get_external_value_content(ctx, retrieved); + assert_ctx_ok(); + ASSERT_EQ(&content, content_ptr); + + nix_gc_decref(ctx, ext); +} + +TEST_F(nix_api_expr_test, nix_get_external_value_content_null) +{ + ASSERT_EQ(nullptr, nix_get_external_value_content(ctx, nullptr)); +} + } // namespace nixC diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 01d15744a750..d7f685205ca3 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -119,6 +119,14 @@ TEST_F(nix_api_expr_test, nix_value_set_get_path) ASSERT_EQ(NIX_TYPE_PATH, nix_get_type(ctx, value)); } +TEST_F(nix_api_expr_test, nix_get_external_invalid) +{ + ASSERT_EQ(nullptr, nix_get_external(ctx, nullptr)); + assert_ctx_err(); + ASSERT_EQ(nullptr, nix_get_external(ctx, value)); + assert_ctx_err(); +} + TEST_F(nix_api_expr_test, nix_build_and_init_list_invalid) { ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, nullptr, state, 0)); diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index fc4f6c0702aa..fef732e1e3f5 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -4,47 +4,10 @@ #include "nix/expr/eval-settings.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/tests/capture-logging.hh" #include "nix/expr/tests/libexpr.hh" namespace nix { -class CaptureLogger : public Logger -{ - std::ostringstream oss; - -public: - CaptureLogger() {} - - std::string get() const - { - return oss.str(); - } - - void log(Verbosity lvl, std::string_view s) override - { - oss << s << std::endl; - } - - void logEI(const ErrorInfo & ei) override - { - showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - } -}; - -class CaptureLogging -{ - std::unique_ptr oldLogger; -public: - CaptureLogging() - { - oldLogger = std::move(logger); - logger = std::make_unique(); - } - - ~CaptureLogging() - { - logger = std::move(oldLogger); - } -}; // Testing eval of PrimOp's class PrimOpTest : public LibExprTest @@ -141,11 +104,10 @@ TEST_F(PrimOpTest, deepSeq) TEST_F(PrimOpTest, trace) { - CaptureLogging l; + testing::CaptureLogging l; auto v = eval("builtins.trace \"test string 123\" 123"); ASSERT_THAT(v, IsIntEq(123)); - auto text = (dynamic_cast(logger.get()))->get(); - ASSERT_NE(text.find("test string 123"), std::string::npos); + ASSERT_THAT(l.get(), ::testing::HasSubstr("test string 123")); } TEST_F(PrimOpTest, placeholder) diff --git a/src/libexpr-tests/value/context.cc b/src/libexpr-tests/value/context.cc index 544f03f3f2e3..c2c5cd3e8bbc 100644 --- a/src/libexpr-tests/value/context.cc +++ b/src/libexpr-tests/value/context.cc @@ -120,8 +120,6 @@ TEST(NixStringContextElemTest, built_built_xp) NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -#ifndef COVERAGE - RC_GTEST_PROP(NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { ExperimentalFeatureSettings xpSettings; @@ -129,6 +127,4 @@ RC_GTEST_PROP(NixStringContextElemTest, prop_round_rip, (const NixStringContextE RC_ASSERT(o == NixStringContextElem::parse(o.to_string(), xpSettings)); } -#endif - } // namespace nix diff --git a/src/libexpr-tests/value/print.cc b/src/libexpr-tests/value/print.cc index 654a50b0ae0b..0082b6eac75e 100644 --- a/src/libexpr-tests/value/print.cc +++ b/src/libexpr-tests/value/print.cc @@ -6,8 +6,6 @@ namespace nix { -using namespace testing; - struct ValuePrintingTests : LibExprTest { template diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index c57b90112886..d512baa92493 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -5,6 +5,10 @@ namespace nix { +void AttrPathNotFound::anchor() {} + +void NoPositionInfo::anchor() {} + static Strings parseAttrPath(std::string_view s) { Strings res; @@ -51,7 +55,7 @@ std::vector AttrPath::resolve(EvalState & state) const } std::pair -findAlongAttrPath(EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn) +findAlongAttrPath(EvalState & state, const std::string & attrPath, const Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 92b67f6ad25e..08412c7be9ca 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -5,7 +5,7 @@ namespace nix { -Bindings Bindings::emptyBindings; +const constinit Bindings Bindings::emptyBindings; /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings @@ -13,7 +13,8 @@ Bindings Bindings::emptyBindings; Bindings * EvalMemory::allocBindings(size_t capacity) { if (capacity == 0) - return &Bindings::emptyBindings; + /* Swear that we are not going to modify this. */ + return const_cast(&Bindings::emptyBindings); if (capacity > std::numeric_limits::max()) throw Error("attribute set of size %d is too big", capacity); stats.nrAttrsets++; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a419530c6dd8..7422cd095bc6 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -10,6 +10,8 @@ namespace nix::eval_cache { +void CachedEvalError::anchor() {} + CachedEvalError::CachedEvalError(ref cursor, Symbol attr) : CloneableError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) , cursor(cursor) @@ -106,7 +108,7 @@ struct AttrDb } template - AttrId doSQLite(F && fun) + AttrId doSQLite(const F & fun) { if (failed) return 0; @@ -124,13 +126,23 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::FullAttrs) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::FullAttrs) + .apply(0, false) + .exec(); AttrId rowId = state->db.getLastInsertedRowId(); assert(rowId); for (auto & attr : attrs) - state->insertAttribute.use()(rowId)(symbols[attr])(AttrType::Placeholder) (0, false).exec(); + state->insertAttribute.use() + .apply(rowId) + .apply(symbols[attr]) + .apply(AttrType::Placeholder) + .apply(0, false) + .exec(); return rowId; }); @@ -150,10 +162,20 @@ struct AttrDb ctx.append(elem->view()); first = false; } - state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx) + state->insertAttributeWithContext.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::String) + .apply(s) + .apply(ctx) .exec(); } else { - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::String) (s).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::String) + .apply(s) + .exec(); } return state->db.getLastInsertedRowId(); @@ -165,7 +187,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Bool) (b ? 1 : 0).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Bool) + .apply(b ? 1 : 0) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -176,7 +203,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Int) (n).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Int) + .apply(n) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -187,9 +219,11 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute - .use()(key.first)(symbols[key.second])( - AttrType::ListOfStrings) (dropEmptyInitThenConcatStringsSep("\t", l)) + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::ListOfStrings) + .apply(dropEmptyInitThenConcatStringsSep("\t", l)) .exec(); return state->db.getLastInsertedRowId(); @@ -201,7 +235,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Placeholder) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Placeholder) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -212,7 +251,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Missing) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Missing) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -223,7 +267,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Misc) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Misc) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -234,7 +283,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Failed) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Failed) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -244,7 +298,7 @@ struct AttrDb { auto state(_state->lock()); - auto queryAttribute(state->queryAttribute.use()(key.first)(symbols[key.second])); + auto queryAttribute(state->queryAttribute.use().apply(key.first).apply(symbols[key.second])); if (!queryAttribute.next()) return {}; @@ -257,7 +311,7 @@ struct AttrDb case AttrType::FullAttrs: { // FIXME: expensive, should separate this out. std::vector attrs; - auto queryAttributes(state->queryAttributes.use()(rowId)); + auto queryAttributes(state->queryAttributes.use().apply(rowId)); while (queryAttributes.next()) attrs.emplace_back(symbols.create(queryAttributes.getStr(0))); return {{rowId, attrs}}; @@ -563,6 +617,7 @@ string_t AttrCursor::getStringWithContext() [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; }, }, c.raw); + root->state.store->addTempRoot(path); if (!root->state.store->isValidPath(path)) { valid = false; break; diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 45cb1e409dd1..ba72b05e98a6 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,11 +1,12 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" +#include "nix/store/store-api.hh" namespace nix { InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) - : CloneableError(state, "path '%s' is not valid", path.to_string()) + : CloneableError(state, "path '%s' is not valid", state.store->printStorePath(path)) , path{path} { } @@ -96,7 +97,7 @@ void EvalErrorBuilder::debugThrow() auto error = std::move(this->error); delete this; - throw error; + throw std::move(error); } template @@ -122,4 +123,32 @@ template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; +void EvalBaseError::anchor() {} + +void ParseError::anchor() {} + +void EvalError::anchor() {} + +void AssertionError::anchor() {} + +void ThrownError::anchor() {} + +void Abort::anchor() {} + +void TypeError::anchor() {} + +void UndefinedVarError::anchor() {} + +void MissingArgumentError::anchor() {} + +void InfiniteRecursionError::anchor() {} + +void StackOverflowError::anchor() {} + +void InvalidPathError::anchor() {} + +void IFDError::anchor() {} + +void RecoverableEvalError::anchor() {} + } // namespace nix diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 9344b0405b17..5bdcd7984644 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -74,6 +74,21 @@ static inline void initGCReal() GC_set_oom_fn(oomHandler); + /* Funnel boehm warnings into debug logs. */ + GC_set_warn_proc([](char * msg, GC_word word) noexcept { + std::array buffer{}; + auto res = snprintf(buffer.data(), buffer.size(), msg, word); + /* Ignore garbage. */ + if (res < 0) + return; + + try { + debug("%s", chomp(std::string_view(buffer.data(), std::min(res, buffer.size() - 1)))); + } catch (...) { + /* Swallow all errors. */ + } + }); + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 5cf0ae04304e..c438ff6c9fb9 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -7,6 +7,8 @@ namespace nix { +void EvalSettings::anchor() {} + void DeprecatedWarnSetting::assign(const bool & v) { value = v; @@ -70,7 +72,7 @@ Strings EvalSettings::parseNixPath(const std::string & s) } EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) - : readOnlyMode{readOnlyMode} + : readOnlyMode{&readOnlyMode} , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_ABORT_ON_WARN"); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d66b8ead1c9..7c92649b68e3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,6 +6,7 @@ #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" #include "nix/util/exit.hh" +#include "nix/util/signals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" #include "nix/util/environment-variables.hh" @@ -277,6 +278,8 @@ EvalState::EvalState( mounted fetchTree. */ auto accessor = settings.pureEval ? storeFS.cast() : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); + /* Cache positive lstat/readlink results to speed up resolveSymlinks. */ + accessor = makeCachingSourceAccessor(accessor); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) @@ -293,15 +296,20 @@ EvalState::EvalState( , internalFS(make_ref()) , derivationInternal{internalFS->addFile( CanonPath("derivation-internal.nix"), -#include "primops/derivation.nix.gen.hh" - )} + { +#embed "primops/derivation.nix" + })} + , importedDrvToDerivation{internalFS->addFile( + CanonPath("imported-drv-to-derivation.nix"), + { +#embed "imported-drv-to-derivation.nix" + })} , store(store) , buildStore(buildStore ? buildStore : store) , inputCache(fetchers::InputCache::create()) , debugRepl(nullptr) , debugStop(false) , trylevel(0) - , srcToStore(make_ref()) , importResolutionCache(make_ref()) , fileEvalCache(make_ref()) , positionToDocComment(make_ref()) @@ -357,8 +365,9 @@ EvalState::EvalState( corepkgsFS->addFile( CanonPath("fetchurl.nix"), -#include "fetchurl.nix.gen.hh" - ); + { +#embed "fetchurl.nix" + }); createBaseEnv(settings); @@ -661,27 +670,34 @@ std::optional EvalState::getDoc(Value & v) return {}; } +static StaticEnv::Vars lexicographicOrder(const SymbolTable & st, StaticEnv::Vars vars) +{ + std::ranges::sort(vars, [&st](const auto & lhs, const auto & rhs) { + return std::string_view(st[lhs.first]) < std::string_view(st[rhs.first]); + }); + return vars; +} + // just for the current level of StaticEnv, not the whole chain. -void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) +static void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { std::cout << ANSI_MAGENTA; - for (auto & i : se.vars) - std::cout << st[i.first] << " "; + for (auto & [name, displacement] : lexicographicOrder(st, se.vars)) + std::cout << st[name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } // just for the current level of Env, not the whole chain. -void printWithBindings(const SymbolTable & st, const Env & env) +static void printWithBindings(const SymbolTable & st, const Env & env) { if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - auto j = env.values[0]->attrs()->begin(); - while (j != env.values[0]->attrs()->end()) { - std::cout << st[j->name] << " "; - ++j; - } + auto * bindings = env.values[0]->attrs(); + /* TODO: Don't print the whole attribute set, since it can be quite large. */ + for (const Attr * attr : bindings->lexicographicOrder(st)) + std::cout << st[attr->name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } @@ -702,7 +718,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << ANSI_MAGENTA; // for the top level, don't print the double underscore ones; // they are in builtins. - for (auto & i : se.vars) + for (auto & i : lexicographicOrder(st, se.vars)) if (!hasPrefix(st[i.first], "__")) std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; @@ -1086,6 +1102,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) return &v; } +namespace { + /** * A helper `Expr` class to lets us parse and evaluate Nix expressions * from a thunk, ensuring that every file is parsed/evaluated only @@ -1129,6 +1147,8 @@ struct ExprParseFile : Expr, gc } }; +} // namespace + void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { auto resolvedPath = getConcurrent(*importResolutionCache, path); @@ -1171,7 +1191,8 @@ void EvalState::resetFileCache() importResolutionCache->clear(); fileEvalCache->clear(); inputCache->clear(); - positions.clear(); + lookupPathResolved->clear(); + rootFS->invalidateCache(); } void EvalState::eval(Expr * e, Value & v) @@ -1341,7 +1362,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) sort = true; } - bindings.bindings->pos = pos; + /* FIXME: Currently we can't track the positions of empty bindings. A way + to fix this is to store the position in the Value storage with more + clever bitpacking (we have spare 32 bits in the Bindings * variant). */ + if (bindings.bindings != &Bindings::emptyBindings) + bindings.bindings->pos = pos; v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted()); } @@ -2208,6 +2233,8 @@ void EvalState::handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, c Value * recovery = nullptr; try { std::rethrow_exception(e); + } catch (const Interrupted & e) { + recovery = allocValue(); } catch (const RecoverableEvalError & e) { recovery = allocValue(); } catch (...) { @@ -2225,6 +2252,8 @@ void EvalState::handleEvalExceptionForApp(Value & v, const Value & savedApp) Value * recovery = nullptr; try { std::rethrow_exception(e); + } catch (const Interrupted & e) { + recovery = allocValue(); } catch (const RecoverableEvalError & e) { recovery = allocValue(); } catch (...) { @@ -2576,23 +2605,16 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - auto dstPathCached = getConcurrent(*srcToStore, path); - - auto dstPath = dstPathCached ? *dstPathCached : [&]() { - auto dstPath = fetchToStore( - fetchSettings, - *store, - path.resolveSymlinks(SymlinkResolution::Ancestors), - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, - path.baseName(), - ContentAddressMethod::Raw::NixArchive, - nullptr, - repair); - allowPath(dstPath); - srcToStore->try_emplace(path, dstPath); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); - return dstPath; - }(); + auto dstPath = fetchToStore( + fetchSettings, + *store, + path.resolveSymlinks(SymlinkResolution::Ancestors), + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, + path.baseName(), + ContentAddressMethod::Raw::NixArchive, + nullptr, + repair); + allowPath(dstPath); context.insert(NixStringContextElem::Opaque{.path = dstPath}); return dstPath; @@ -3266,14 +3288,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ continue; auto r = *rOpt; - auto res = (r / CanonPath(suffix)).resolveSymlinks(); - if (res.pathExists()) + auto suffixPath = CanonPath(suffix); + if (auto cachedRes = getConcurrent(*rOpt->resolvedPaths, suffixPath)) { + if (*cachedRes) + return **cachedRes; + else + // Cached negative lookup. + continue; + } + + auto res = (r.path / suffixPath).resolveSymlinks(); + if (res.pathExists()) { + r.resolvedPaths->emplace(suffixPath, res); return res; + } // Backward compatibility hack: throw an exception if access // to this path is not allowed. if (auto accessor = res.accessor.dynamic_pointer_cast()) accessor->checkAccess(res.path); + + // Cache negative lookups too. + r.resolvedPaths->emplace(suffixPath, std::nullopt); } if (hasPrefix(path, "nix/")) @@ -3287,17 +3323,22 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ .debugThrow(); } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::shared_ptr +EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; if (auto cached = getConcurrent(*lookupPathResolved, value)) return *cached; - auto finish = [&](std::optional res) { - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else + auto finish = [&](std::optional maybePath) { + std::shared_ptr res; + if (maybePath) { + debug("resolved search path element '%s' to '%s'", value, *maybePath); + res = std::make_shared( + *maybePath, make_ref()); + } else { debug("failed to resolve search path element '%s'", value); + } lookupPathResolved->emplace(std::string(value), res); return res; }; @@ -3442,7 +3483,7 @@ void forceNoNullByte(std::string_view s, std::function pos) if (pos) { error.atPos(pos()); } - throw error; + throw std::move(error); } } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 693b1946ee46..6416713f7b3d 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -162,14 +162,14 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) - throw errMsg; + throw std::move(errMsg); Outputs result; for (auto elem : outTI->listView()) { if (elem->type() != nString) - throw errMsg; + throw std::move(errMsg); auto out = outputs.find(elem->string_view()); if (out == outputs.end()) - throw errMsg; + throw std::move(errMsg); result.insert(*out); } return result; @@ -393,7 +393,7 @@ static void getDerivations( EvalState & state, Value & vIn, const std::string & pathPrefix, - Bindings & autoArgs, + const Bindings & autoArgs, PackageInfos & drvs, Done & done, bool ignoreAssertionFailures) @@ -464,7 +464,7 @@ void getDerivations( EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, + const Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures) { diff --git a/src/libexpr/include/nix/expr/attr-path.hh b/src/libexpr/include/nix/expr/attr-path.hh index fd48705b8b7b..ab765ec1b1a2 100644 --- a/src/libexpr/include/nix/expr/attr-path.hh +++ b/src/libexpr/include/nix/expr/attr-path.hh @@ -12,7 +12,7 @@ MakeError(AttrPathNotFound, Error); MakeError(NoPositionInfo, Error); std::pair -findAlongAttrPath(EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn); +findAlongAttrPath(EvalState & state, const std::string & attrPath, const Bindings & autoArgs, Value & vIn); /** * Heuristic to find the filename and lineno or a nix value. diff --git a/src/libexpr/include/nix/expr/attr-set.hh b/src/libexpr/include/nix/expr/attr-set.hh index 4d3821feda97..38d654282a79 100644 --- a/src/libexpr/include/nix/expr/attr-set.hh +++ b/src/libexpr/include/nix/expr/attr-set.hh @@ -29,11 +29,15 @@ struct Attr Symbol name; PosIdx pos; Value * value = nullptr; + Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name) , pos(pos) - , value(value) {}; - Attr() {}; + , value(value) + { + } + + constexpr Attr() {} auto operator<=>(const Attr & a) const { @@ -70,7 +74,7 @@ public: * An instance of bindings objects with 0 attributes. * This object must never be modified. */ - static Bindings emptyBindings; + static const constinit Bindings emptyBindings; private: /** @@ -101,7 +105,7 @@ private: */ Attr attrs[0]; - Bindings() = default; + constexpr Bindings() = default; Bindings(const Bindings &) = delete; Bindings(Bindings &&) = delete; Bindings & operator=(const Bindings &) = delete; @@ -550,14 +554,14 @@ public: Value & alloc(std::string_view name, PosIdx pos = noPos); - Bindings * finish() + const Bindings * finish() { bindings->sort(); finishSizeIfNecessary(); return bindings; } - Bindings * alreadySorted() + const Bindings * alreadySorted() { finishSizeIfNecessary(); return bindings; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 68c8f6543f59..4a360970ba21 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -44,7 +44,7 @@ NIX_DECLARE_CONFIG_SERIALISER(Diagnose) * @throws The error returned by mkError if level is `Fatal` and mkError returns a value */ template -void diagnose(const Setting & setting, F && mkError) +void diagnose(const Setting & setting, const F & mkError) { auto withError = [&](bool fatal, auto && handler) { auto maybeError = mkError(fatal); @@ -64,7 +64,7 @@ void diagnose(const Setting & setting, F && mkError) withError(false, [](auto && error) { logWarning(error.info()); }); return; case Diagnose::Fatal: - withError(true, [](auto && error) { throw std::move(error); }); + withError(true, [](auto && error) { throw std::forward(error); }); return; } } diff --git a/src/libexpr/include/nix/expr/eval-cache.hh b/src/libexpr/include/nix/expr/eval-cache.hh index d2ead2bb4614..0feb7d5649ff 100644 --- a/src/libexpr/include/nix/expr/eval-cache.hh +++ b/src/libexpr/include/nix/expr/eval-cache.hh @@ -16,6 +16,9 @@ class AttrCursor; struct CachedEvalError : CloneableError { +private: + void anchor() override; +public: const ref cursor; const Symbol attr; diff --git a/src/libexpr/include/nix/expr/eval-error.hh b/src/libexpr/include/nix/expr/eval-error.hh index 68aa7b0643a2..8dce750e073c 100644 --- a/src/libexpr/include/nix/expr/eval-error.hh +++ b/src/libexpr/include/nix/expr/eval-error.hh @@ -23,6 +23,9 @@ class EvalBaseError : public CloneableError { template friend class EvalErrorBuilder; + + void anchor() override; + public: EvalState & state; @@ -61,8 +64,11 @@ MakeError(InfiniteRecursionError, EvalError); * Inherits from EvalBaseError (not EvalError) because resource exhaustion * should not be cached. */ -struct StackOverflowError : public CloneableError +class StackOverflowError : public CloneableError { + void anchor() override; + +public: StackOverflowError(EvalState & state) : CloneableError(state, "stack overflow; max-call-depth exceeded") { @@ -79,8 +85,10 @@ MakeError(IFDError, EvalBaseError); */ MakeError(RecoverableEvalError, EvalBaseError); -struct InvalidPathError : public CloneableError +class InvalidPathError : public CloneableError { + void anchor() override; + public: StorePath path; diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index d9dba95370b1..33ae8509970f 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -43,6 +43,10 @@ public: struct EvalSettings : Config { +private: + void anchor() override; + +public: /** * Function used to interpret look path entries of a given scheme. * @@ -71,7 +75,14 @@ struct EvalSettings : Config EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); - bool & readOnlyMode; + /* FIXME: This really shouldn't be public. The C API should have non-global settings instead. */ + bool * readOnlyMode = nullptr; + + bool isReadOnly() const + { + assert(readOnlyMode); + return *readOnlyMode; + } static Strings getDefaultNixPath(); @@ -191,6 +202,10 @@ struct EvalSettings : Config - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime) - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) + + As a result, every fetch must be a []{#pure-fetch}*pure fetch* — one that references immutable content: + [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) and [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) require a locked revision, and [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) and [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) require a `sha256` hash. + A mutable reference, such as a Git branch or tag without a revision, is rejected, since its result could otherwise change over time. )"}; Setting traceImportFromDerivation{ diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 89e2d5099da2..86abb3c8f2ae 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -226,7 +226,7 @@ struct StaticEvalSymbols line, column, functor, toString, right, wrong, structuredAttrs, json, allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites, maxSize, maxClosureSize, builder, args, contentAddressed, impure, outputHash, outputHashAlgo, outputHashMode, recurseForDerivations, description, self, epsilon, startSet, - operator_, key, path, prefix, outputSpecified; + operator_, key, path, prefix, outputSpecified, requiredSystemFeatures; Expr::AstSymbols exprSymbols; @@ -279,6 +279,7 @@ struct StaticEvalSymbols .path = alloc.create("path"), .prefix = alloc.create("prefix"), .outputSpecified = alloc.create("outputSpecified"), + .requiredSystemFeatures = alloc.create("requiredSystemFeatures"), .exprSymbols = { .sub = alloc.create("__sub"), .lessThan = alloc.create("__lessThan"), @@ -396,6 +397,7 @@ public: const ref internalFS; const SourcePath derivationInternal; + const SourcePath importedDrvToDerivation; /** * Store used to materialise .drv files. @@ -407,8 +409,6 @@ public: */ const ref buildStore; - RootValue vImportedDrvToDerivation = nullptr; - const ref inputCache; /** @@ -460,10 +460,6 @@ public: private: - /* Cache for calls to addToStore(); maps source paths to the store - paths. */ - const ref> srcToStore; - /** * A cache that maps paths to "resolved" paths for importing Nix * expressions, i.e. `/foo` to `/foo/default.nix`. @@ -489,7 +485,15 @@ private: LookupPath lookupPath; - const ref, StringViewHash, std::equal_to<>>> + struct LookupPathResolvedState + { + SourcePath path; + const ref>> resolvedPaths; + }; + + const ref< + boost:: + concurrent_flat_map, StringViewHash, std::equal_to<>>> lookupPathResolved; /** @@ -626,9 +630,10 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt`. + * If it is not found, return `nullptr`. */ - std::optional resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); + std::shared_ptr + resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); /** * Evaluate an expression to normal form @@ -729,6 +734,26 @@ public: std::optional tryAttrsToString( const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + enum class CopyLazyPaths : bool { + PreserveLazy = false, + Copy = true, + }; + + /** + * For efficiency reasons, some store paths (as seen by the evaluator) in + * the storeFS at their content-addressed locations don't get copied to the + * store eagerly. This saves on needless I/O and possibly IPC if all the + * evaluator does is just evaluate nix expressions from those locations. + * This function copies such store objects to the store if they aren't already valid. + */ + void ensureLazyPathCopied(const StorePath & path); + + /** + * Ensure that all NixStringContextElem::Opaque context elements get fetched + * to the store. + */ + void ensureLazyPathsCopied(const NixStringContext & context); + /** * String coercion. * @@ -1035,9 +1060,14 @@ public: /** * Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`. + * @param copyLazyPaths When encountering a lazy path (i.e. a string with Opaque context that's also "mounted" on + * the storeFS), fetch the store path to the store. */ SourcePath realisePath( - const PosIdx pos, Value & v, std::optional resolveSymlinks = SymlinkResolution::Full); + const PosIdx pos, + Value & v, + std::optional resolveSymlinks = SymlinkResolution::Full, + CopyLazyPaths copyLazyPaths = CopyLazyPaths::PreserveLazy); /** * Realise the given string with context, and return the string with outputs instead of downstream output diff --git a/src/libexpr/include/nix/expr/fetch-tree.hh b/src/libexpr/include/nix/expr/fetch-tree.hh new file mode 100644 index 000000000000..3eb8a01c0c5f --- /dev/null +++ b/src/libexpr/include/nix/expr/fetch-tree.hh @@ -0,0 +1,18 @@ +#pragma once + +#include "nix/expr/eval.hh" + +namespace nix { + +/** + * Convert a libfetchers `Input` to libexpr `Value`. + */ +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/get-drvs.hh b/src/libexpr/include/nix/expr/get-drvs.hh index 4beccabe2ad3..8e25c25e1aa7 100644 --- a/src/libexpr/include/nix/expr/get-drvs.hh +++ b/src/libexpr/include/nix/expr/get-drvs.hh @@ -112,7 +112,7 @@ void getDerivations( EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, + const Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures); diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 4213476fe73a..7334b42f8f17 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,6 +20,7 @@ headers = [ config_pub_h ] + files( 'eval-profiler.hh', 'eval-settings.hh', 'eval.hh', + 'fetch-tree.hh', 'function-trace.hh', 'gc-small-vector.hh', 'get-drvs.hh', diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 07fbed403c2f..b13c00ad541e 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -557,7 +557,7 @@ public: std::numeric_limits::max()); if (pos) err.atPos(positions[pos]); - throw err; + throw std::move(err); } std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart); }; diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index f9bd06589e42..2482d53ea041 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -89,7 +89,7 @@ public: * @see https://github.com/NixOS/nix/issues/14642 */ template - void visit(F && f) + void visit(const F & f) { std::visit( overloaded{ diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index 7e44a6b66ebc..07fcc337e355 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -17,6 +17,12 @@ class EvalState; * * See: https://github.com/NixOS/nix/issues/9730 */ -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth = 0); +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context = nullptr, + size_t depth = 0); } // namespace nix diff --git a/src/libexpr/include/nix/expr/print.hh b/src/libexpr/include/nix/expr/print.hh index 229f7159d15a..8e6d0f9bf09a 100644 --- a/src/libexpr/include/nix/expr/print.hh +++ b/src/libexpr/include/nix/expr/print.hh @@ -10,6 +10,7 @@ #include #include "nix/util/fmt.hh" +#include "nix/expr/value/context.hh" #include "nix/expr/print-options.hh" namespace nix { @@ -64,7 +65,12 @@ bool isReservedKeyword(const std::string_view str); */ std::ostream & printIdentifier(std::ostream & o, std::string_view s); -void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions{}); +void printValue( + EvalState & state, + std::ostream & str, + Value & v, + PrintOptions options = PrintOptions{}, + NixStringContext * context = nullptr); /** * A partially-applied form of `printValue` which can be formatted using `<<` @@ -77,12 +83,15 @@ private: EvalState & state; Value & value; PrintOptions options; + NixStringContext * context; public: - ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions{}) + ValuePrinter( + EvalState & state, Value & value, PrintOptions options = PrintOptions{}, NixStringContext * context = nullptr) : state(state) , value(value) , options(options) + , context(context) { } }; diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index b2d0f9295ed5..e1c7725624f5 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -432,9 +432,35 @@ struct ValueBase Value * const * elems; }; - struct Failed : gc_cleanup + /** + * Wrapper that stores a std::exception_ptr on the GC heap with a finaliser + * that runs the exception_ptr destructor (which is refcounted internally). + * This is not a part of the Failed structure to avoid cycles with finalisers, + * which Boehm warns about. + */ + struct ExceptionRef : gc_cleanup { + ExceptionRef(std::exception_ptr ex) + : ex(std::move(ex)) + { + assert(this->ex); + } + + ExceptionRef(ExceptionRef &&) = delete; + ExceptionRef(const ExceptionRef &) = delete; + ExceptionRef & operator=(ExceptionRef &&) = delete; + ExceptionRef & operator=(const ExceptionRef &) = delete; + + /* To appease -Wweak-vtables. */ + virtual ~ExceptionRef(); + std::exception_ptr ex; + }; + + struct Failed : gc + { + ExceptionRef * exRef; + /** * Optional value for recovering `RecoverableEvalError` * Must be set iff `ex` is an instance of `RecoverableEvalError`. @@ -442,16 +468,15 @@ struct ValueBase Value * recoveryValue; Failed(std::exception_ptr ex, Value * recoveryValue) - : ex(ex) + : exRef(new /* ExceptionRef : gc_cleanup */ ExceptionRef(ex)) , recoveryValue(recoveryValue) { - assert(this->ex); } [[noreturn]] void rethrow() const { try { - std::rethrow_exception(ex); + std::rethrow_exception(exRef->ex); } catch (BaseError & e) { /* Rethrow the copy of the exception - not the original one. Stack tracing mechanisms rely on being able to modify the exceptions @@ -480,7 +505,7 @@ struct PayloadTypeToInternalType MACRO(ValueBase::StringWithContext, string, tString) \ MACRO(ValueBase::Path, path, tPath) \ MACRO(ValueBase::Null, null_, tNull) \ - MACRO(Bindings *, attrs, tAttrs) \ + MACRO(const Bindings *, attrs, tAttrs) \ MACRO(ValueBase::List, bigList, tListN) \ MACRO(ValueBase::SmallList, smallList, tListSmall) \ MACRO(ValueBase::ClosureThunk, thunk, tThunk) \ @@ -875,7 +900,7 @@ protected: primOp = std::bit_cast(payload[1]); } - void getStorage(Bindings *& attrs) const noexcept + void getStorage(const Bindings *& attrs) const noexcept { Payload payload = loadPayload(); attrs = std::bit_cast(payload[1]); @@ -938,7 +963,7 @@ protected: setSingleDWordPayload(std::bit_cast(primOp)); } - void setStorage(Bindings * bindings) noexcept + void setStorage(const Bindings * bindings) noexcept { setSingleDWordPayload(std::bit_cast(bindings)); } @@ -1336,7 +1361,7 @@ public: setStorage(Null{}); } - inline void mkAttrs(Bindings * a) noexcept + inline void mkAttrs(const Bindings * a) noexcept { setStorage(a); } @@ -1462,7 +1487,7 @@ public: const Bindings * attrs() const noexcept { - return getStorage(); + return getStorage(); } const PrimOp * primOp() const noexcept diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index 31f03addf2a0..8fc948622768 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -11,6 +11,8 @@ namespace nix { class BadNixStringContextElem final : public CloneableError { + void anchor() override; + public: std::string_view raw; diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 08393435b48c..d218a0beaefc 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -9,6 +9,8 @@ using json = nlohmann::json; namespace nix { +namespace { + // for more information, refer to // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp class JSONSax : nlohmann::json_sax @@ -200,6 +202,8 @@ class JSONSax : nlohmann::json_sax } }; +} // namespace + void parseJSON(EvalState & state, const std::string_view & s_, Value & v) { JSONSax parser(state, v); @@ -208,4 +212,6 @@ void parseJSON(EvalState & state, const std::string_view & s_, Value & v) throw JSONParseError("Invalid JSON Value"); } +void JSONParseError::anchor() {} + } // namespace nix diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index 59f6f6f70dfd..085ac7df4698 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -26,3 +26,5 @@ void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc->beginOffset = loc->endOffset; loc->endOffset += len; } + +nix::Parser::~Parser() {} diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 510b6d696e3b..6016fa81a79e 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -144,16 +144,6 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) -subdir('nix-meson-build-support/generate-header') - -generated_headers = [] -foreach header : [ - 'imported-drv-to-derivation.nix', - 'fetchurl.nix', -] - generated_headers += gen_header.process(header) -endforeach - sources = files( 'attr-path.cc', 'attr-set.cc', @@ -240,7 +230,6 @@ this_library = library( config_priv_h, parser_tab[1], lexer_tab[1], - generated_headers, soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, diff --git a/src/libexpr/parser-scanner-decls.hh b/src/libexpr/parser-scanner-decls.hh index e4e06188334c..1bec23837b99 100644 --- a/src/libexpr/parser-scanner-decls.hh +++ b/src/libexpr/parser-scanner-decls.hh @@ -11,7 +11,9 @@ namespace nix { class Parser : public parser::BisonParser { +public: using BisonParser::BisonParser; + ~Parser() override; }; } // namespace nix diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index ca303208173e..4ceab2b4dbfe 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -22,10 +22,55 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +void EvalState::ensureLazyPathCopied(const StorePath & path) +{ + if (settings.isReadOnly()) + return; + + auto mount = storeFS->getMount(CanonPath(store->printStorePath(path))); + if (!mount) + return; + + /* TODO: We could memoise this in-memory if necessary. */ + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + /* Force a copy. mountInput does a dryRun to just calculate the storePath and narHash. */ + FetchMode::Copy, + path.name()); + + /* This can happen if the source gets modified by another process while we are evaluaing + from it. Alternatively, the caching might be unsound and fetcher cache is poisoned somehow. + See https://github.com/NixOS/nix/issues/14317. */ + if (storePath != path) { + throw Error( + (unsigned int) 102, + "store path ('%1%') was hashed to avoid a full copy at first, but upon reading it again, the contents have changed ('%2%'), so we can not proceed. Make sure files do not change during evaluation", + store->printStorePath(path), + store->printStorePath(storePath)); + } +} + +void EvalState::ensureLazyPathsCopied(const NixStringContext & context) +{ + for (const auto & c : context) + if (auto * o = std::get_if(&c.raw)) + /* TODO: This could be done in parallel. */ + ensureLazyPathCopied(o->path); +} + StorePath EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) { - auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + /* To mount the input, dryRun is sufficient. We still compute the narHash (to check for mismatches) and the store + path to figure out where to mount it. TODO: This could be relaxed in the future by making outPath and narHash + lazier. Good code that doesn't do `toString ./.` or otherwise inspects the outPath string and only uses it for + doing relative imports does not even require computing the store path. That is a big invasive change though and + would require having a special "LazyStorePathString" thunk. narHash also doesn't need to be computed eagerly in + case it's not actually specified (like during local development with a dirty tree) - in that case narHash could + also become a lazy app/thunk that shares the state with the storePath delayed computation. */ + auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1ff6..8e01ce73639d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,10 @@ #include "nix/store/names.hh" #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" +#include "nix/util/configuration.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/store/build.hh" +#include "nix/util/strings.hh" #include "nix/util/util.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" @@ -63,7 +67,7 @@ std::string EvalState::realiseString(Value & s, StorePathSet * storePathsOutMayb nix::NixStringContext stringContext; auto rawStr = coerceToString(pos, s, stringContext, "while realising a string").toOwned(); auto rewrites = realiseContext(stringContext, storePathsOutMaybe, isIFD); - + ensureLazyPathsCopied(stringContext); return nix::rewriteStrings(rawStr, rewrites); } @@ -88,7 +92,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + /* If the path happens to be mounted on the storeFS, that means it's lazy path string and would get + copied to the store on-demand (when referenced in a derivation). The string is equal to final + store path where the store object would end up (the path is hashed before mounting). */ + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -121,7 +129,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS buildReqs.reserve(drvs.size()); for (auto & d : drvs) buildReqs.emplace_back(DerivedPath{d}); - buildStore->buildPaths(buildReqs, bmNormal, store); + buildStore->getBuilder(store)->buildPaths(buildReqs, bmNormal); StorePathSet outputsToCopyAndAllow; @@ -158,7 +166,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS return res; } -SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional resolveSymlinks) +SourcePath EvalState::realisePath( + const PosIdx pos, Value & v, std::optional resolveSymlinks, CopyLazyPaths copyLazyPaths) { NixStringContext context; @@ -167,6 +176,8 @@ SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optionalmkAttrs(attrs); - if (!state.vImportedDrvToDerivation) { - state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); - state.eval( - state.parseExprFromString( -#include "imported-drv-to-derivation.nix.gen.hh" - , state.rootPath(CanonPath::root)), - **state.vImportedDrvToDerivation); - } + auto vImportedDrvToDerivation = state.allocValue(); + state.evalFile(state.importedDrvToDerivation, *vImportedDrvToDerivation); // has caching - state.forceFunction( - **state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); - v.mkApp(*state.vImportedDrvToDerivation, w); - state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); + v.mkApp(vImportedDrvToDerivation, w); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix"); } /** @@ -441,10 +444,11 @@ static RegisterPrimOp primop_import( /* !!! Should we pass the Pos or the file name too? */ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); -/* Load a ValueInitializer from a DSO and return whatever it initializes */ +/* Load a ValueInitializer from a DSO and return whatever it initializes. FIXME: This doesn't + work with chroot stores. */ void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v) { - auto path = state.realisePath(pos, *args[0]); + auto path = state.realisePath(pos, *args[0], SymlinkResolution::Full, EvalState::CopyLazyPaths::Copy); std::string sym( state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); @@ -471,7 +475,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value /* We don't dlclose because v may be a primop referencing a function in the shared object file */ } -/* Execute a program and parse its output */ +/* Execute a program and parse its output. FIXME: This doesn't work with chroot stores. */ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v) { state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.exec"); @@ -509,6 +513,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v) .debugThrow(); } + state.ensureLazyPathsCopied(context); auto output = runProgram(program, true, toOsStrings(std::move(commandArgs))); Expr * parsed; try { @@ -694,7 +699,7 @@ static RegisterPrimOp primop_isPath({ }); template -static inline void withExceptionContext(Trace trace, Callable && func) +static inline void withExceptionContext(Trace trace, const Callable & func) { try { func(); @@ -976,6 +981,7 @@ static RegisterPrimOp primop_break( } // Return the value we were passed. + state.forceValue(*args[0], pos); v = *args[0]; }}); @@ -1037,7 +1043,35 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value ** a static RegisterPrimOp primop_addErrorContext( PrimOp{ .name = "__addErrorContext", + .args = {"context", "value"}, .arity = 2, + .doc = R"( + Evaluate *context*, which can be coerced to a string, + and append it to any error or stack traces displayed while evaluating *value*. + Then return *value*. + + This function is useful for providing helpful context in complex Nix expressions + when the evaluation of *value* fails. + The additional context is applied when evaluating *value* itself fails, + not when attributes or elements of *value* are evaluated. + + For example, the module system from nixpkgs uses this to show + the relevant information about the options that were evaluating + when an error occurs. + + ```nix-repl + nix-repl> addErrorContext "while evaluating foo" (throw "bar") + error: + … while evaluating foo + + … while calling the 'throw' builtin + at «string»:1:56: + 1| with builtins; addErrorContext "while evaluating foo" (throw "bar") + | ^ + + error: bar + ``` + )", // The normal trace item is redundant .addTrace = false, .impl = prim_addErrorContext, @@ -1483,13 +1517,15 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ - Derivation drv; - drv.name = drvName; + Derivation drv{ + .name = std::string{drvName}, + }; NixStringContext context; bool contentAddressed = false; bool isImpure = false; + bool isSubmittingOutputs = false; std::optional outputHash; std::optional outputHashAlgo; std::optional ingestionMethod; @@ -1503,6 +1539,17 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); + // Like `warn`, but with the position of the attribute and the derivation name as an added trace. + auto warnAttr = [&](HintFmt msg) { + ErrorInfo info{ + .level = lvlWarn, + .msg = std::move(msg), + .pos = state.positions[i->pos], + }; + info.traces.push_back(Trace{.hint = HintFmt{"while evaluating derivation '%1%'", drvName}}); + logWarning(info); + }; + auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" @@ -1612,40 +1659,36 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName handleOutputs(ss); break; } + case EvalState::s.requiredSystemFeatures.getId(): { + /* Only parsed to detect `builder-rpc-v0`; skip + entirely unless the experimental feature is + enabled. */ + if (!experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations)) + break; + state.forceList(*i->value, pos, context_below); + for (auto elem : i->value->listView()) { + auto name = state.forceString(*elem, context, pos, context_below); + if (name == drvFeatureBuilderRpcV0) { + isSubmittingOutputs = true; + break; + } + } + break; + } default: break; } switch (i->name.getId()) { case EvalState::s.allowedReferences.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", - drvName); - break; case EvalState::s.allowedRequisites.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", - drvName); - break; case EvalState::s.disallowedReferences.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", - drvName); - break; case EvalState::s.disallowedRequisites.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", - drvName); - break; case EvalState::s.maxSize.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", - drvName); - break; case EvalState::s.maxClosureSize.getId(): - warn( - "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", - drvName); + warnAttr(HintFmt( + "'structuredAttrs' disables the effect of the derivation attribute '%1%'; use 'outputChecks..%1%' instead", + key)); break; default: break; @@ -1653,10 +1696,18 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); + + /* Re-interpret the attribute's value as a list of + strings. + + We may wish to warn here better future-compat + later, e.g. requiring that it be a list of + strings without spaces to begin with. */ + auto forceStringList = [&] { return tokenizeString(s); }; + if (i->name == state.s.json) { - warn( - "In derivation '%s': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.", - drvName); + warnAttr(HintFmt( + "setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.")); drv.structuredAttrs = StructuredAttrs::parse(s); } else { drv.env.emplace(key, s); @@ -1677,8 +1728,22 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName handleHashMode(s); break; case EvalState::s.outputs.getId(): - handleOutputs(tokenizeString(s)); + handleOutputs(forceStringList()); + break; + case EvalState::s.requiredSystemFeatures.getId(): { + /* Only parsed to detect `builder-rpc-v0`; skip + entirely unless the experimental feature is + enabled. */ + if (!experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations)) + break; + for (auto & name : forceStringList()) { + if (name == drvFeatureBuilderRpcV0) { + isSubmittingOutputs = true; + break; + } + } break; + } default: break; } @@ -1717,16 +1782,19 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName StorePathSet refs; state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { - drv.inputSrcs.insert(j); + drv.inputs.srcs.insert(j); if (j.isDerivation()) { - drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames(); + drv.inputs.drvs.map[j].value = state.store->readDerivation(j).outputNames(); } } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); + drv.inputs.drvs.ensureSlot(*b.drvPath).value.insert(b.output); + }, + [&](const NixStringContextElem::Opaque & o) { + state.ensureLazyPathCopied(o.path); + drv.inputs.srcs.insert(o.path); }, - [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); }, }, c.raw); } @@ -1772,7 +1840,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName }, }; - drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + if (!isSubmittingOutputs) + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); drv.outputs.insert_or_assign("out", std::move(dof)); } @@ -1784,7 +1853,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { - drv.env[i] = hashPlaceholder(i); + if (!isSubmittingOutputs) + drv.env[i] = hashPlaceholder(i); if (isImpure) drv.outputs.insert_or_assign( i, @@ -1927,21 +1997,21 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V .debugThrow(); NixStringContext context; - auto path = - state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'") - .path; + SourcePath sourcePath = state.coerceToPath( + pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'"); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true).string()); - if (!state.store->isInStore(path.abs())) - state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); - auto path2 = state.store->toStorePath(path.abs()).first; - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(NixStringContextElem::Opaque{.path = path2}); - v.mkString(path.abs(), context, state.mem); + if (!state.store->isStorePath(sourcePath.path.abs())) + sourcePath = sourcePath.resolveSymlinks(SymlinkResolution::Full); + if (!state.store->isInStore(sourcePath.path.abs())) + state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); + auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) + state.store->getBuilder()->ensurePath(storePath); + context.insert(NixStringContextElem::Opaque{.path = storePath}); + v.mkString(sourcePath.path.abs(), context, state.mem); } static RegisterPrimOp primop_storePath({ @@ -2666,9 +2736,10 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu StorePathSet refs; for (auto c : context) { - if (auto p = std::get_if(&c.raw)) + if (auto p = std::get_if(&c.raw)) { + state.ensureLazyPathCopied(p->path); refs.insert(p->path); - else + } else state .error( "files created by %1% may not reference derivations, but %2% references %3%", @@ -3051,11 +3122,7 @@ static RegisterPrimOp primop_attrNames({ alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrNames, }); @@ -3089,11 +3156,7 @@ static RegisterPrimOp primop_attrValues({ Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrValues, }); @@ -3120,9 +3183,7 @@ static RegisterPrimOp primop_getAttr({ the `.` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_getAttr, }); @@ -3212,9 +3273,7 @@ static RegisterPrimOp primop_hasAttr({ `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_hasAttr, }); @@ -3275,12 +3334,7 @@ static RegisterPrimOp primop_removeAttrs({ evaluates to `{ y = 2; }`. - # Time Complexity - - O(n + k log k) where: - - n = number of attributes in input set - k = number of attribute names to remove + Has `O(n + k log k)` time complexity, where `n` is number of attributes in the *set* and `k` is the size of *list*. )", .impl = prim_removeAttrs, }); @@ -3369,9 +3423,7 @@ static RegisterPrimOp primop_listToAttrs({ { foo = 123; bar = 456; } ``` - # Time Complexity - - O(n log n) where n = number of list elements + Has `O(n log n)` time complexity, where `n` is size of the list. )", .impl = prim_listToAttrs, }); @@ -3448,12 +3500,7 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - # Time Complexity - - O(n * log m) where: - - n = number of attributes in the smaller set - m = number of attributes in the larger set + Has `O(n log m)` time complexity, where `n` and `m` are the sizes of the smallest and largest set respectively. )", .impl = prim_intersectAttrs, }); @@ -3494,12 +3541,7 @@ static RegisterPrimOp primop_catAttrs({ evaluates to `[1 2]`. - # Time Complexity - - O(n * log m) where: - - n = list length - m = number of attributes per set + Has `O(n)` time complexity, where `n` is the size of the *list*. )", .impl = prim_catAttrs, }); @@ -3544,9 +3586,7 @@ static RegisterPrimOp primop_functionArgs({ the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. - # Time Complexity - - O(n) where n = number of formal arguments + Has constant time complexity. )", .impl = prim_functionArgs, }); @@ -3580,13 +3620,9 @@ static RegisterPrimOp primop_mapAttrs({ evaluates to `{ a = 10; b = 20; }`. - # Time Complexity - - O(n) where: - - n = number of attributes - - Calls to `f` are performed afterwards, when needed. + Has `O(n)` time complexity, where `n` is the size of the *attrset*. + Note that no calls to *f* are performed by the builtin. + The function *f* is called on demand when a resulting attribute value is evaluated. )", .impl = prim_mapAttrs, }); @@ -3675,12 +3711,7 @@ static RegisterPrimOp primop_zipAttrsWith({ } ``` - # Time Complexity - - O(N * log k) where: - - N = total attributes across all sets - k = number of unique keys across all sets + Has `O(n log n)` time complexity, where `n` is the number of attributes across all sets. )", .impl = prim_zipAttrsWith, }); @@ -3748,9 +3779,7 @@ static RegisterPrimOp primop_head({ isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. - # Time Complexity - - O(1) + Has constant time complexity. )", .impl = prim_head, }); @@ -3782,10 +3811,6 @@ static RegisterPrimOp primop_tail({ > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. - - # Time Complexity - - O(n) where n = list length (copies n-1 elements) )", .impl = prim_tail, }); @@ -3821,13 +3846,9 @@ static RegisterPrimOp primop_map({ evaluates to `[ "foobar" "foobla" "fooabc" ]`. - # Time Complexity - - O(n) where: - - n = list length - - Calls to `f` are performed afterwards when needed. + Has `O(n)` time complexity, where `n` is the size of the *list*. + Note that no calls to *f* are performed by the builtin, but *f* itself is evaluated and its type is checked eagerly. + The function *f* is called on demand when a resulting list element is evaluated. )", .impl = prim_map, }); @@ -3877,13 +3898,7 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. - - # Time Complexity - - O(n * T_f) (eager; predicate is forced) where: - - n = list length - T_f = predicate evaluation time + Has linear time complexity in the size of the input *list*. )", .impl = prim_filter, }); @@ -3907,15 +3922,7 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. - - # Time Complexity - - O(n * T) (worst case) where: - - n = list length - T = time to compare two elements - - returns early if the elements is found + Short-circuits and does not evaluate elements that occur in the list after the first match. )", .impl = prim_elem, }); @@ -3933,12 +3940,6 @@ static RegisterPrimOp primop_concatLists({ .args = {"lists"}, .doc = R"( Concatenate a list of lists into a single list. - - # Time Complexity - - O(N) where: - - N = total number of elements across all lists )", .impl = prim_concatLists, }); @@ -3955,10 +3956,6 @@ static RegisterPrimOp primop_length({ .args = {"e"}, .doc = R"( Return the length of the list *e*. - - # Time Complexity - - O(1) )", .impl = prim_length, }); @@ -3991,24 +3988,40 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. + e.g. + ```nix + foldl' op nul [ x0 x1 x2 ] + = + let + strictly = f: a: builtins.seq a (f a); + y0 = op nul x0; + y1 = strictly op y0 x1; + y2 = strictly op y1 x2; + in + y2 + + # and, ignoring strictness/laziness + == + op (op (op nul x0) x1) x2 + ``` For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. The first argument of `op` is the accumulator whereas the second - argument is the current element being processed. The return value - of each application of `op` is evaluated immediately, even for - intermediate values. + argument is the current element being processed. - # Time Complexity + The return value of each application of `op` is evaluated immediately, + even for intermediate values. + This way, `foldl'` can operate in constant stack space, allowing it to operate on large lists, + regardless of [max-call-depth](@docroot@/command-ref/conf-file.md#conf-max-call-depth). - O(n * T_op) where: + Conventionally, a fold function without the `'` ("prime") preserves laziness, + but lacks these benefits. + See also [Nixpkgs `lib.foldl`](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.lists.foldl). - n = list length - T_op = `op` call evaluation time + Has linear time complexity in the size of the list. )", .impl = prim_foldlStrict, }); @@ -4047,15 +4060,7 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_pred) where: - - - n = `list` length - - T_pred = `pred` call evaluation time - - returns early when `pred` returns `true` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `true`. )", .impl = prim_any, }); @@ -4071,15 +4076,7 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_f) where: - - - n = list length - - T_f = predicate evaluation time - - returns early when `pred` returns `false` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `false`. )", .impl = prim_all, }); @@ -4119,16 +4116,7 @@ static RegisterPrimOp primop_genList({ returns the list `[ 0 1 4 9 16 ]`. - # Time Complexity - - Complexity of `genList generator n`: O(n) - - Complexity of `deepSeq (genList generator n)`: O(n * T_f) - - where: - - n = requested length - T_f = `generator` call evaluation time + Has linear time complexity. )", .impl = prim_genList, }); @@ -4240,15 +4228,8 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. - # Time Complexity - - O(n log n * T_cmp), where: - - n = `list` length - T_cmp = `comparator` call evaluation time - - Uses an adaptive sort that exploits existing sorted runs in the input, - down to O(n * T_cmp) when the list is already sorted. + Runs in `O(n log n)` time on average, where `n` is the size of the *list*. + Uses an adaptive sort that exploits existing sorted runs in the input, down to `O(n)` when the list is already sorted. )", .impl = prim_sort, }); @@ -4311,12 +4292,7 @@ static RegisterPrimOp primop_partition({ { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` - # Time Complexity - - O(n * T_pred) where: - - n = list length - T_pred = `pred` call evaluation time + Runs in linear time in the size of the *list*. )", .impl = prim_partition, }); @@ -4371,13 +4347,7 @@ static RegisterPrimOp primop_groupBy({ { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` - # Time Complexity - - O(N * T_f + N * log k) where: - - N = number of `list` elements - T_f = `f` call evaluation time - k = number of unique groups + Has `O(n log n)` time complexity, where `n` is the size of the input *list*. )", .impl = prim_groupBy, }); @@ -4420,14 +4390,6 @@ static RegisterPrimOp primop_concatMap({ .doc = R"( This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. - - # Time Complexity - - O(k * T_f + N) where: - - k = length of input list - T_f = time to call `f` on an element - N = total number of elements returned by `f` calls )", .impl = prim_concatMap, }); @@ -5131,13 +5093,6 @@ static RegisterPrimOp primop_concatStringsSep({ Concatenate a list of strings with a separator between each element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. - - # Time Complexity - - O(n + m) (amortized) where: - - n = number of list elements - m = total length of output string )", .impl = prim_concatStringsSep, }); @@ -5223,13 +5178,7 @@ static RegisterPrimOp primop_replaceStrings({ evaluates to `"fabir"`. - # Time Complexity - - O(n * k * c) (worst case) where: - - n = length of input string - k = number of replacement patterns - c = average length of patterns in 'from' list + Has `O(n k)` time complexity, where `n` is the length of *s* and `k` is the number of replacements. )", .impl = prim_replaceStrings, }); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index d5d5de0b9aaf..7eef9daa2560 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -2,6 +2,7 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/globals.hh" namespace nix { @@ -272,7 +273,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg state.error("context key '%s' is not a store path", name).atPos(i.pos).debugThrow(); auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(namePath); + state.store->getBuilder()->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); if (auto attr = i.value->attrs()->get(sPath)) { diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index dbb8c2186889..1d17c24ef11a 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,5 +1,5 @@ -# This is the implementation of the ‘derivation’ builtin function. -# It's actually a wrapper around the ‘derivationStrict’ primop. +# This is the implementation of the `derivation` builtin function. +# It's actually a wrapper around the `derivationStrict` primop. # Note that the following comment will be shown in :doc in the repl, but not in the manual. /** @@ -46,7 +46,7 @@ let outputToAttrListElement = outputName: { name = outputName; value = commonAttrs // { - outPath = builtins.getAttr outputName strict; + outPath = strict.${outputName}; drvPath = strict.drvPath; type = "derivation"; inherit outputName; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index db182ab499b2..fe3873c101b4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -23,6 +23,8 @@ static void runFetchClosureWithRewrite( const std::optional & toPathMaybe, Value & v) { + if (toPathMaybe) + state.store->addTempRoot(*toPathMaybe); // establish toPath or throw @@ -74,6 +76,7 @@ static void runFetchClosureWithRewrite( static void runFetchClosureWithContentAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); @@ -103,6 +106,7 @@ static void runFetchClosureWithContentAddressedPath( static void runFetchClosureWithInputAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index afd61e90fbb5..6ffcf91ed30a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,8 +1,11 @@ +#include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" #include "nix/fetchers/registry.hh" @@ -19,6 +22,101 @@ namespace nix { +/** + * Adapter for putting libfetchers data into a thunk closure. + * Used as the argument to prim_forceLazyFetcherAttr in a lazy apply thunk. + */ +class LazyFetcherAttr : public ExternalValueBase, public gc_cleanup +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + fetchers::LazyAttr lazy; + +public: + LazyFetcherAttr(fetchers::LazyAttr lazy) + : lazy(std::move(lazy)) + { + } + + fetchers::ResolvedAttr force() + { + return lazy->compute(); + } + +protected: + std::ostream & print(std::ostream & str) const override + { + unreachable(); + } + +public: + std::string showType() const override + { + unreachable(); + } + + std::string typeOf() const override + { + unreachable(); + } +}; + +void LazyFetcherAttr::anchor() {} + +/** + * Initialize a `Value` from a resolved fetcher attribute. + */ +static void resolvedAttrToValue(EvalState & state, Value & v, const fetchers::ResolvedAttr & resolved) +{ + std::visit( + overloaded{ + [&](const std::string & s) { v.mkString(s, state.mem); }, + [&](uint64_t n) { v.mkInt(n); }, + [&](const Explicit & b) { v.mkBool(b.t); }, + }, + resolved); +} + +/** + * internal primop: Force a LazyFetcherAttr external value. + */ +static void prim_forceLazyFetcherAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + Value & arg = *args[0]; + + state.forceValue(arg, pos); + // We only construct this primop with LazyFetcherAttr preapplied. + assert(arg.type() == nExternal); + auto * ext = dynamic_cast(args[0]->external()); + assert(ext); + + resolvedAttrToValue(state, v, ext->force()); +} + +/** + * Emit a lazy thunk for a LazyAttr: mkApp(primop, externalValue). + */ +static void emitLazyAttrThunk(EvalState & state, const fetchers::LazyAttr & lazyAttr, Value & dest) +{ + // not user-callable (unregistered, internal) + static PrimOp forcePrimOp{ + .name = "__forceLazyFetcherAttr", + .arity = 1, + .impl = prim_forceLazyFetcherAttr, + .internal = true, + }; + + auto * vExt = state.allocValue(); + vExt->mkExternal(new LazyFetcherAttr(lazyAttr)); + + auto * vPrimOp = state.allocValue(); + vPrimOp->mkPrimOp(&forcePrimOp); + + dest.mkApp(vPrimOp, vExt); +} + void emitTreeAttrs( EvalState & state, const StorePath & storePath, @@ -51,7 +149,9 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); } - if (auto revCount = input.getRevCount()) + if (auto revCount = maybeGetLazyAttr(input.attrs, "revCount")) + emitLazyAttrThunk(state, *revCount, attrs.alloc("revCount")); + else if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) attrs.alloc("revCount").mkInt(0); @@ -463,7 +563,7 @@ static void fetch( // Try to get the path from the local store or substituters try { - state.store->ensurePath(expectedPath); + state.store->getBuilder()->ensurePath(expectedPath); debug("using substituted/cached path '%s' for '%s'", state.store->printStorePath(expectedPath), *url); state.allowAndSetStorePathString(expectedPath, v); return; @@ -476,35 +576,39 @@ static void fetch( } } - // Download the file/tarball if substitution failed or no hash was provided - auto storePath = unpack ? fetchToStore( - state.fetchSettings, - *state.store, - fetchers::downloadTarball(*state.store, state.fetchSettings, *url), - FetchMode::Copy, - name) - : fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; - - if (expectedHash) { - auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashPath( - {state.store->requireStoreObjectAccessor(storePath)}, - FileSerialisationMethod::Flat, - HashAlgorithm::SHA256) - .hash; - if (hash != *expectedHash) { - state - .error( - "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, - expectedHash->to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)) - .withExitStatus(102) - .debugThrow(); + if (unpack) { + auto attrs = fetchers::Attrs{ + {"type", "tarball"}, + {"url", *url}, + {"name", name}, + }; + if (expectedHash) + attrs.emplace("narHash", expectedHash->to_string(HashFormat::SRI, true)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); + auto cachedInput = + state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); + state.mkStorePathString(storePath, v); + } else { + auto storePath = fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; + if (expectedHash) { + auto hash = hashPath( + {state.store->requireStoreObjectAccessor(storePath)}, + FileSerialisationMethod::Flat, + HashAlgorithm::SHA256) + .hash; + if (hash != *expectedHash) + state + .error( + "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", + *url, + expectedHash->to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)) + .withExitStatus(102) + .debugThrow(); } + state.allowAndSetStorePathString(storePath, v); } - - state.allowAndSetStorePathString(storePath, v); } static void prim_fetchurl(EvalState & state, const PosIdx pos, Value ** args, Value & v) diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index b8abc6409af9..c49755970525 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -1,8 +1,3 @@ -generated_headers += gen_header.process( - 'derivation.nix', - preserve_path_from : meson.project_source_root(), -) - sources += files( 'context.cc', 'fetchClosure.cc', diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ed91cad85a47..b0a5224f26e6 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -7,7 +7,13 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth) +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context, + size_t depth) { checkInterrupt(); @@ -22,6 +28,8 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setlexicographicOrder(state.symbols)) { str << state.symbols[i->name] << " = "; - printAmbiguous(state, *i->value, str, seen, depth + 1); + printAmbiguous(state, *i->value, str, seen, context, depth + 1); str << "; "; } str << "}"; @@ -52,7 +60,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set seen; size_t totalAttrsPrinted = 0; size_t totalListItemsPrinted = 0; @@ -577,9 +578,12 @@ class Printer printBool(v); break; - case nString: + case nString: { printString(v); + if (context) + copyContext(v, *context); break; + } case nPath: printPath(v); @@ -632,10 +636,11 @@ class Printer } public: - Printer(std::ostream & output, EvalState & state, PrintOptions options) + Printer(std::ostream & output, EvalState & state, PrintOptions options, NixStringContext * context) : output(output) , state(state) , options(options) + , context(context) { } @@ -656,14 +661,14 @@ class Printer } }; -void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options) +void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options, NixStringContext * context) { - Printer(output, state, options).print(v); + Printer(output, state, options, context).print(v); } std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) { - printValue(printer.state, output, printer.value, printer.options); + printValue(printer.state, output, printer.value, printer.options, printer.context); return output; } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 8d48a48391cc..c844cd143032 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -104,6 +104,8 @@ json printValueAsJSON( return out; } +void JSONSerializationError::anchor() {} + void printValueAsJSON( EvalState & state, bool strict, diff --git a/src/libexpr/value.cc b/src/libexpr/value.cc index 8dbb277750ba..bfdee3cded9a 100644 --- a/src/libexpr/value.cc +++ b/src/libexpr/value.cc @@ -8,6 +8,8 @@ namespace nix { +Value::ExceptionRef::~ExceptionRef() {} + Value Value::vEmptyList = []() { Value res; res.setStorage(List{.size = 0, .elems = nullptr}); diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 60ba5352077d..c61e1681c473 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,6 +4,8 @@ namespace nix { +void BadNixStringContextElem::anchor() {} + NixStringContextElem NixStringContextElem::parse(std::string_view s0, const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index db415d9173e7..c55f4c8b1bda 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,16 +50,33 @@ headers += files('nix_api_fetchers.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixfetchersc', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'soversion' : nix_soversion, + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixfetchersc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixfetchersc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libfetchers-c/meson.options b/src/libfetchers-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libfetchers-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libfetchers-c/nix_api_fetchers.cc b/src/libfetchers-c/nix_api_fetchers.cc index 7fefedb0c70f..2b3dd56631c5 100644 --- a/src/libfetchers-c/nix_api_fetchers.cc +++ b/src/libfetchers-c/nix_api_fetchers.cc @@ -7,7 +7,7 @@ extern "C" { nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context) { try { - auto fetchersSettings = nix::make_ref(nix::fetchers::Settings{}); + auto fetchersSettings = nix::make_ref(); return new nix_fetchers_settings{ .settings = fetchersSettings, }; diff --git a/src/libfetchers-c/package.nix b/src/libfetchers-c/package.nix index 9a601d70417c..f490c91e7cb2 100644 --- a/src/libfetchers-c/package.nix +++ b/src/libfetchers-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libfetchers-tests/attrs.cc b/src/libfetchers-tests/attrs.cc new file mode 100644 index 000000000000..4d0cbbb9ef74 --- /dev/null +++ b/src/libfetchers-tests/attrs.cc @@ -0,0 +1,75 @@ +#include + +#include "nix/fetchers/attrs.hh" + +#include + +namespace nix::fetchers { + +TEST(LazyAttr, resolveToInt) +{ + Attrs attrs; + attrs.insert_or_assign( + "count", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(42); + }}))); + EXPECT_EQ(maybeGetIntAttr(attrs, "count"), 42); +} + +TEST(LazyAttr, resolveToString) +{ + Attrs attrs; + attrs.insert_or_assign( + "name", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("hello"); + }}))); + EXPECT_EQ(maybeGetStrAttr(attrs, "name"), "hello"); +} + +TEST(LazyAttr, resolveToBool) +{ + Attrs attrs; + attrs.insert_or_assign( + "flag", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return Explicit{true}; + }}))); + EXPECT_EQ(maybeGetBoolAttr(attrs, "flag"), true); +} + +TEST(LazyAttr, attrsToJSONForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "x", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(99); + }}))); + auto json = attrsToJSON(attrs); + EXPECT_EQ(json["x"], 99); +} + +TEST(LazyAttr, attrsToQueryForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "v", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("val"); + }}))); + auto query = attrsToQuery(attrs); + EXPECT_EQ(query.at("v"), "val"); +} + +TEST(LazyAttr, notCalledUntilForced) +{ + int calls = 0; + Attrs attrs; + attrs.insert_or_assign( + "lazy", LazyAttr(make_ref(LazyAttrComputation{.compute = [&calls]() -> ResolvedAttr { + calls++; + return uint64_t(1); + }}))); + EXPECT_EQ(calls, 0); + maybeGetIntAttr(attrs, "lazy"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git-lfs-fetch.cc b/src/libfetchers-tests/git-lfs-fetch.cc new file mode 100644 index 000000000000..d01a43151df5 --- /dev/null +++ b/src/libfetchers-tests/git-lfs-fetch.cc @@ -0,0 +1,28 @@ +#include "nix/fetchers/git-lfs-fetch.hh" +#include "nix/util/url.hh" + +#include + +namespace nix::lfs { + +struct GitLFSParameterizedTestFixture : public ::testing::TestWithParam> +{}; + +TEST_P(GitLFSParameterizedTestFixture, get_lfs_api) +{ + auto & [input, expected] = GetParam(); + ASSERT_EQ(getLfsApi(parseURL(input)).endpoint, expected); +}; + +INSTANTIATE_TEST_SUITE_P( + GitLFSTests, + GitLFSParameterizedTestFixture, + ::testing::Values( + std::pair{"https://git-server.com/foo/bar", "https://git-server.com/foo/bar.git/info/lfs"}, + std::pair{"https://git-server.com/foo/bar.git", "https://git-server.com/foo/bar.git/info/lfs"}, + std::pair{"https://git-server.com", "https://git-server.com/.git/info/lfs"}, + std::pair{"https://git-server.com/", "https://git-server.com/.git/info/lfs"}, + std::pair{"https://git-server.com//", "https://git-server.com//.git/info/lfs"}, + std::pair{"https://git-server.com/foo/bar/", "https://git-server.com/foo/bar.git/info/lfs"})); + +} // namespace nix::lfs diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 580769936d41..c1b357e12a0a 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -1,5 +1,7 @@ #include "nix/fetchers/git-utils.hh" #include "nix/util/file-system.hh" +#include "nix/util/tests/gmock-matchers.hh" + #include #include #include @@ -19,7 +21,7 @@ namespace nix::fetchers { class GitUtilsTest : public ::testing::Test { // We use a single repository for all tests. - std::unique_ptr delTmpDir; + AutoDelete delTmpDir; protected: std::filesystem::path tmpDir; @@ -27,27 +29,19 @@ class GitUtilsTest : public ::testing::Test public: void SetUp() override { - tmpDir = createTempDir(); - delTmpDir = std::make_unique(tmpDir, true); - - // Create the repo with libgit2 - git_libgit2_init(); - git_repository * repo = nullptr; - auto r = git_repository_init(&repo, tmpDir.string().c_str(), 0); - ASSERT_EQ(r, 0); - git_repository_free(repo); + tmpDir = createTempDir() / "test-git-repo"; + GitRepo::openRepo(tmpDir, {.create = true}); + delTmpDir = AutoDelete(tmpDir, true); } void TearDown() override { - // Destroy the AutoDelete, triggering removal - // not AutoDelete::reset(), which would cancel the deletion. - delTmpDir.reset(); + delTmpDir.deletePath(); } ref openRepo() { - return GitRepo::openRepo(tmpDir, {.create = true}); + return GitRepo::openRepo(tmpDir, {.create = false}); } std::string getRepoName() const @@ -92,13 +86,24 @@ TEST_F(GitUtilsTest, sink_basic) auto result = repo->dereferenceSingletonDirectory(sink->flush()); auto accessor = repo->getAccessor(result, {}, getRepoName()); - auto entries = accessor->readDirectory(CanonPath::root); - ASSERT_EQ(entries.size(), 5u); - ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); - ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); - ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); - ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0u); - ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); + + ASSERT_THAT( + accessor, + testing::HasDirectory( + CanonPath::root, + std::set{ + "hello", + "bye", + "bye-link", + "empty", + "links", + })); + + ASSERT_THAT(accessor, testing::HasContents(CanonPath("hello"), "hello world")); + ASSERT_THAT(accessor, testing::HasContents(CanonPath("bye"), "thanks for all the fish")); + ASSERT_THAT(accessor, testing::HasSymlink(CanonPath("bye-link"), "bye")); + ASSERT_THAT(accessor, testing::HasDirectory(CanonPath("empty"), std::set{})); + ASSERT_THAT(accessor, testing::HasContents(CanonPath("links/foo"), "hello world")); }; TEST_F(GitUtilsTest, sink_hardlink) @@ -117,12 +122,197 @@ TEST_F(GitUtilsTest, sink_hardlink) sink->flush(); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("does not exist")); - ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); - ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + ASSERT_THAT(e.msg(), ::testing::HasSubstr("does not exist")); + ASSERT_THAT(e.msg(), ::testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), ::testing::HasSubstr("foo-1.1/link")); } }; +TEST_F(GitUtilsTest, sink_no_parent_dir) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->createRegularFile(CanonPath("foo/bar"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "boom", /*executable=*/false); + }); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); +} + +TEST_F(GitUtilsTest, sink_no_parent_dir_symlink) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->createSymlink(CanonPath("foo/bar"), "target"); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); +} + +TEST_F(GitUtilsTest, sink_no_parent_dir_hardlink) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->createHardlink(CanonPath("foo/bar"), CanonPath("foo")); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("parent of 'foo/bar' is not a directory"))); +} + +TEST_F(GitUtilsTest, sink_replacing_empty_directory) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createDirectory(CanonPath("foo/bar")); + /* Under tarball unpacking semantics, creating the same directories + (implicitly or explicitly) is fine. */ + sink->createDirectory(CanonPath("foo/bar")); + sink->createDirectory(CanonPath("foo")); + + sink->createRegularFile(CanonPath("foo/bar"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + auto accessor = repo->getAccessor(sink->flush(), {}, getRepoName()); + + ASSERT_THAT(accessor, testing::HasDirectory(CanonPath("foo"), std::set{"bar"})); + ASSERT_THAT(accessor, testing::HasContents(CanonPath("foo/bar"), "test")); +} + +TEST_F(GitUtilsTest, sink_replacing_non_empty_directory) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createDirectory(CanonPath("foo/bar")); + + /* This fails. libarchive (and other tarball unpackers) doesn't recursive unlink existing non-empty + directories. + https://github.com/libarchive/libarchive/blob/761652401fe35fca9744607a0cf0009afbf04f42/libarchive/archive_write_disk_posix.c#L3411-L3417 + */ + + sink->createRegularFile(CanonPath("foo"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->flush(); + }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("cannot create 'foo', conflicting non-empty directory"))); +} + +TEST_F(GitUtilsTest, sink_hardlink_to_directory) +{ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createHardlink(CanonPath("bar"), CanonPath("foo")); + + sink->flush(); + }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("cannot create a hard link to a directory"))); +} + +TEST_F(GitUtilsTest, sink_hardlink_to_directory_root) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createDirectory(CanonPath("foo")); + sink->createHardlink(CanonPath("bar"), CanonPath::root); + + auto accessor = repo->getAccessor(sink->flush(), {}, getRepoName()); + + /* FIXME: Why does it behave this way? This seems like a bug. */ + ASSERT_THAT( + accessor, + testing::HasDirectory( + CanonPath::root, + std::set{ + "foo", + })); +} + +TEST_F(GitUtilsTest, sink_hardlink_to_self) +{ + /* Here we are more strict than libarchive, which only warns on cyclic hardlinks. + https://github.com/libarchive/libarchive/blob/761652401fe35fca9744607a0cf0009afbf04f42/libarchive/archive_write_disk_posix.c#L632-L641 + */ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath::root); + sink->createHardlink(CanonPath("foo"), CanonPath("foo")); + + sink->flush(); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("/foo"))); +} + +TEST_F(GitUtilsTest, sink_non_directory_root) +{ + /* FIXME: Allow non-directory roots. GitFileSystemObjectSink is too tarball-brained. */ + ASSERT_THAT( + [&]() { + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createRegularFile(CanonPath::root, [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "test", /*executable=*/false); + }); + + sink->flush(); + }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("cannot create a file at the root of the git repository"))); +} + TEST_F(GitUtilsTest, peel_reference) { // Create a commit in the repo diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956b9..7123df54ebbe 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -33,6 +33,9 @@ deps_private += rapidcheck gtest = dependency('gtest', main : true) deps_private += gtest +gmock = dependency('gmock') +deps_private += gmock + libgit2 = dependency('libgit2') deps_private += libgit2 @@ -40,6 +43,8 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', + 'attrs.cc', + 'git-lfs-fetch.cc', 'git-utils.cc', 'git.cc', 'input.cc', diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index cc9e72af460d..f3ecc3a87bc0 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -4,6 +4,18 @@ namespace nix::fetchers { +ResolvedAttr forceAttr(const Attr & attr) +{ + return std::visit( + overloaded{ + [](const LazyAttr & lazy) -> ResolvedAttr { return lazy->compute(); }, + [](const std::string & v) -> ResolvedAttr { return v; }, + [](uint64_t v) -> ResolvedAttr { return v; }, + [](const Explicit & v) -> ResolvedAttr { return v; }, + }, + attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; @@ -26,11 +38,12 @@ nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { json[attr.first] = v->t; } else unreachable(); @@ -38,12 +51,23 @@ nlohmann::json attrsToJSON(const Attrs & attrs) return json; } +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) + return {}; + if (auto v = std::get_if(&i->second)) + return *v; + return {}; +} + std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } @@ -61,7 +85,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not an integer", name); } @@ -79,7 +104,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if>(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if>(&resolved)) return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -96,11 +122,12 @@ StringMap attrsToQuery(const Attrs & attrs) { StringMap query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, fmt("%d", *v)); - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, *v); - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); } else unreachable(); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index cf60e29a8e15..390180080145 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -26,6 +26,8 @@ create table if not exists Cache ( struct CacheImpl : Cache { + void anchor() override; + struct State { SQLite db; @@ -60,7 +62,11 @@ struct CacheImpl : Cache void upsert(const Key & key, const Attrs & value) override { _state.lock() - ->upsert.use()(key.first)(attrsToJSON(key.second).dump())(attrsToJSON(value).dump())(time(nullptr)) + ->upsert.use() + .apply(key.first) + .apply(attrsToJSON(key.second).dump()) + .apply(attrsToJSON(value).dump()) + .apply(time(nullptr)) .exec(); } @@ -87,7 +93,7 @@ struct CacheImpl : Cache auto keyJSON = attrsToJSON(key.second).dump(); - auto stmt(state->lookup.use()(key.first)(keyJSON)); + auto stmt(state->lookup.use().apply(key.first).apply(keyJSON)); if (!stmt.next()) { debug("did not find cache entry for '%s:%s'", key.first, keyJSON); return {}; @@ -156,6 +162,10 @@ struct CacheImpl : Cache } }; +void Cache::anchor() {} + +void CacheImpl::anchor() {} + ref Settings::getCache() const { auto cache(_cache.lock()); diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index f92b94a0b3bd..8839a0fde4f0 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -4,4 +4,6 @@ namespace nix::fetchers { Settings::Settings() {} +void Settings::anchor() {} + } // namespace nix::fetchers diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 3af2d4c83e88..085c97da7d01 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -3,8 +3,23 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/environment-variables.hh" +#include + namespace nix { +struct SrcToStore +{ + boost::concurrent_flat_map< + std::tuple, + std::tuple> + cache; +}; + +ref fetchers::Settings::createSrcToStore() +{ + return make_ref(); +} + fetchers::Cache::Key makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path) { @@ -36,6 +51,14 @@ std::pair fetchToStore2( PathFilter * filter, RepairFlag repair) { + auto srcToStoreKey = std::make_tuple(path, method.raw, std::string(name)); + + if (!filter) { + auto dstPathCached = getConcurrent(settings.srcToStore->cache, srcToStoreKey); + if (dstPathCached && (mode == FetchMode::DryRun || std::get<2>(*dstPathCached) == FetchMode::Copy)) + return std::make_pair(std::get<0>(*dstPathCached), std::get<1>(*dstPathCached)); + } + std::optional cacheKey; auto [subpath, fingerprint] = filter ? std::pair>{path.path, std::nullopt} @@ -59,6 +82,7 @@ std::pair fetchToStore2( path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode)); return {storePath, hash}; } debug("source path '%s' not in store", path); @@ -67,7 +91,6 @@ std::pair fetchToStore2( static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; if (barf && !filter) throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); - // FIXME: could still provide in-memory caching keyed on `SourcePath`. debug("source path '%s' is uncacheable", path); } @@ -82,6 +105,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun ? [&]() { + // FIXME: we may have already computed this above. auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( @@ -102,8 +126,9 @@ std::pair fetchToStore2( throw Error("path '%s' lacks a CA field", store.printStorePath(storePath)); info->ca->hash; }); - debug( - "copied '%s' to '%s' (hash '%s')", + printMsg( + lvlChatty, + "copied source '%s' -> '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); @@ -113,6 +138,9 @@ std::pair fetchToStore2( if (cacheKey) settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); + if (!filter) + settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode)); + return {storePath, hash}; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index fb87f9b94506..4d6826cce439 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,13 +1,18 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" #include "nix/util/fs-sink.hh" +#include "nix/store/build.hh" #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/url.hh" +#include "nix/util/users.hh" +#include "nix/store/pathlocks.hh" +#include "nix/util/environment-variables.hh" +#include #include namespace nix::fetchers { @@ -121,6 +126,9 @@ std::optional Input::getFingerprint(Store & store) const auto fingerprint = scheme->getFingerprint(store, *this); + if (fingerprint) + fingerprint = std::string(scheme->schemeName()) + ":" + *fingerprint; + cachedFingerprint = fingerprint; return fingerprint; @@ -163,8 +171,7 @@ bool Input::isFinal() const std::optional Input::isRelative() const { - assert(scheme); - return scheme->isRelative(*this); + return scheme ? scheme->isRelative(*this) : std::nullopt; } Attrs Input::toAttrs() const @@ -316,7 +323,9 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings try { auto storePath = computeStorePath(store); - store.ensurePath(storePath); + store.addTempRoot(storePath); + + store.getBuilder()->ensurePath(storePath); debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath)); @@ -342,6 +351,21 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings } } + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. Note that we don't need + * this when substituting the input. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + createDirs(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + lock.setDeletion(true); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto [accessor, result] = scheme->getAccessor(settings, store, *this); if (!accessor->fingerprint) @@ -361,19 +385,20 @@ Input Input::applyOverrides(std::optional ref, std::optional void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const { - assert(scheme); + if (!scheme) + throw Error("cannot clone unsupported input '%s'", attrsToJSON(attrs)); scheme->clone(settings, store, *this, destDir); } std::optional Input::getSourcePath() const { - assert(scheme); - return scheme->getSourcePath(*this); + return scheme ? scheme->getSourcePath(*this) : std::nullopt; } void Input::putFile(const CanonPath & path, std::string_view contents, std::optional commitMsg) const { - assert(scheme); + if (!scheme) + throw Error("unsupported input '%s' does not support modifying file '%s'", attrsToJSON(attrs), path); return scheme->putFile(*this, path, contents, commitMsg); } @@ -504,12 +529,11 @@ std::string publicKeys_to_string(const std::vector & publicKeys) namespace nlohmann { -using namespace nix; - #ifndef DOXYGEN_SKIP -fetchers::PublicKey adl_serializer::from_json(const json & json) +nix::fetchers::PublicKey adl_serializer::from_json(const json & json) { + using namespace nix; fetchers::PublicKey res = {}; auto & obj = getObject(json); if (auto * type = optionalValueAt(obj, "type")) @@ -520,7 +544,7 @@ fetchers::PublicKey adl_serializer::from_json(const json & return res; } -void adl_serializer::to_json(json & json, const fetchers::PublicKey & p) +void adl_serializer::to_json(json & json, const nix::fetchers::PublicKey & p) { json["type"] = p.type; json["key"] = p.key; diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 6fe7d2504ec3..ceb6457d473d 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -62,11 +62,6 @@ std::pair> FilteringSourceAccessor::getFin return next->getFingerprint(prefix / path); } -void FilteringSourceAccessor::invalidateCache(const CanonPath & path) -{ - next->invalidateCache(prefix / path); -} - void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) @@ -75,6 +70,9 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { +private: + void anchor() override {}; +public: SharedSync> allowedPrefixes; boost::concurrent_flat_set allowedPaths; diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index 9d2fb928d603..c80732a0c21f 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -1,12 +1,15 @@ #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-utils.hh" #include "nix/store/filetransfer.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" #include "nix/util/users.hh" #include "nix/util/util.hh" #include "nix/util/hash.hh" +#include "nix/util/json-utils.hh" #include "nix/store/ssh.hh" #include "nix/util/deleter.hh" @@ -22,9 +25,7 @@ namespace nix::lfs { static void downloadToSink( const std::string & url, const std::optional & authHeader, - // FIXME: passing a StringSink is superfluous, we may as well - // return a string. Or use an abstract Sink for streaming. - StringSink & sink, + Sink & sink, std::string sha256Expected, size_t sizeExpected) { @@ -33,29 +34,25 @@ static void downloadToSink( if (authHeader.has_value()) headers.push_back({"Authorization", *authHeader}); request.headers = headers; - getFileTransfer()->download(std::move(request), sink); - auto sizeActual = sink.s.length(); - if (sizeExpected != sizeActual) - throw Error("size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, sizeActual); + HashSink hashSink(HashAlgorithm::SHA256); + TeeSink teeSink(hashSink, sink); - auto sha256Actual = hashString(HashAlgorithm::SHA256, sink.s).to_string(HashFormat::Base16, false); + getFileTransfer()->download(std::move(request), teeSink); + + auto hashResult = hashSink.finish(); + + if (sizeExpected != hashResult.numBytesDigested) + throw Error( + "size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, hashResult.numBytesDigested); + + auto sha256Actual = hashResult.hash.to_string(HashFormat::Base16, false); if (sha256Actual != sha256Expected) throw Error( "hash mismatch while fetching %s: expected sha256:%s but got sha256:%s", url, sha256Expected, sha256Actual); } -namespace { - -struct LfsApiInfo -{ - std::string endpoint; - std::optional authHeader; -}; - -} // namespace - -static LfsApiInfo getLfsApi(const ParsedURL & url) +LfsApiInfo getLfsApi(ParsedURL url) { assert(url.authority.has_value()); if (url.scheme == "ssh") { @@ -95,7 +92,41 @@ static LfsApiInfo getLfsApi(const ParsedURL & url) return {queryResp.at("href").get(), authIt->get()}; } - return {url.to_string() + "/info/lfs", std::nullopt}; + /** + * Try to mimic what git-lfs will do to plain remotes + * https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md + * + * Try to be smarter with remotes ending in a /, like + * `https://github.com/NixOS/nix/`. This should be + * `https://github.com/NixOS/nix.git/info/lfs`, not + * `https://github.com/NixOS/nix/.git/info/lfs` + */ + bool hasDotGit = false; + for (auto it = url.path.rbegin(); it != url.path.rend(); ++it) { + if (it->empty()) + continue; + if (!it->ends_with(".git")) + *it += ".git"; + hasDotGit = true; + break; + } + if (!hasDotGit) { + if (url.path.size() > 1) // e.g. {"", ""} (single trailing slash) + url.path.back() = ".git"; + else if (url.path.size() == 1) // {""} + url.path.push_back(".git"); + else { // {} + url.path.push_back(""); + url.path.push_back(".git"); + } + } + if (url.path.back().empty()) + url.path.back() = "info"; + else + url.path.push_back("info"); + url.path.push_back("lfs"); + + return {url.to_string(), std::nullopt}; } typedef std::unique_ptr> GitConfig; @@ -254,7 +285,7 @@ std::vector Fetch::fetchUrls(const std::vector & pointe void Fetch::fetch( const std::string & content, const CanonPath & pointerFilePath, - StringSink & sink, + Sink & sink, std::function sizeCallback) const { debug("trying to fetch '%s' using git-lfs", pointerFilePath); @@ -278,9 +309,13 @@ void Fetch::fetch( std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false) + "/" + pointer->oid; auto cachePath = cacheDir / key; - if (pathExists(cachePath)) { + AutoCloseFD cacheFile(openFileReadonly(cachePath, FinalSymlink::DontFollow)); + if (cacheFile) { debug("using cache entry %s -> %s", key, PathFmt(cachePath)); - sink(readFile(cachePath)); + FdSource cacheSource(cacheFile.get()); + auto size = getFileSize(cacheFile.get()); + sizeCallback(size); + cacheSource.drainInto(sink, size); return; } debug("did not find cache entry for %s", key); @@ -291,7 +326,8 @@ void Fetch::fetch( const auto obj = objUrls[0]; try { - std::string sha256 = obj.at("oid"); // oid is also the sha256 + // Use the committed pointer's oid/size for integrity, not server's claim + std::string sha256 = pointer->oid; std::string ourl = obj.at("actions").at("download").at("href"); auto authHeader = [&]() -> std::optional { const auto & download = obj.at("actions").at("download"); @@ -303,14 +339,37 @@ void Fetch::fetch( return std::nullopt; return std::string(*authIt); }(); - const uint64_t size = obj.at("size"); - sizeCallback(size); - downloadToSink(ourl, authHeader, sink, sha256, size); + const uint64_t size = pointer->size; + + auto objOid = getString(valueAt(getObject(obj), "oid")); + auto objSize = getUnsigned(valueAt(getObject(obj), "size")); + if (objOid != pointer->oid || objSize != pointer->size) { + throw Error( + "LFS server returned mismatched oid/size for '%s' (got oid=%s size=%d, expected oid=%s size=%d)", + pointerFilePath, + objOid, + objSize, + pointer->oid, + pointer->size); + } debug("creating cache entry %s -> %s", key, PathFmt(cachePath)); + if (!pathExists(cachePath.parent_path())) createDirs(cachePath.parent_path()); - writeFile(cachePath, sink.s); + auto [tempFile, tempPath] = createTempFile(cachePath.parent_path(), {}); + AutoDelete tempDeleter(tempPath); + FdSink tempSink(tempFile.get()); + downloadToSink(ourl, authHeader, tempSink, sha256, size); + tempSink.flush(); + + std::filesystem::rename(tempPath, cachePath); + tempDeleter.cancel(); + + FdSource cacheSource(tempFile.get()); + cacheSource.restart(); + sizeCallback(size); + cacheSource.drainInto(sink, size); debug("%s fetched with git-lfs", pointerFilePath); } catch (const nlohmann::json::out_of_range & e) { diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 6bc474395af8..ac46028cbe0a 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -45,6 +45,7 @@ #include #include #include +#include namespace std { @@ -74,6 +75,8 @@ namespace nix { struct GitSourceAccessor; +namespace { + struct GitError final : public CloneableError { template @@ -97,6 +100,34 @@ struct GitError final : public CloneableError } }; +struct GitIndexerSink final : public BufferedSink +{ + git_indexer * indexer; + git_indexer_progress stats{}; + + GitIndexerSink(git_indexer * indexer) + : BufferedSink(1 * 1024 * 1024) + , indexer(indexer) + { + assert(indexer); + } + + GitIndexerSink(GitIndexerSink &&) = delete; + GitIndexerSink(const GitIndexerSink &) = delete; + GitIndexerSink & operator=(GitIndexerSink &&) = delete; + GitIndexerSink & operator=(const GitIndexerSink &) = delete; + ~GitIndexerSink() = default; + + void writeUnbuffered(std::string_view data) override + { + checkInterrupt(); + if (git_indexer_append(indexer, data.data(), data.size(), &stats)) + throw GitError("appending to git packfile index"); + } +}; + +} // namespace + typedef std::unique_ptr> Repository; typedef std::unique_ptr> TreeEntry; typedef std::unique_ptr> Tree; @@ -226,7 +257,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options) { - if (pathExists(path.string())) + if (pathExists(path)) return; if (!options.create) @@ -333,46 +364,67 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this void flush() override { + std::size_t objectCount; + if (git_mempack_object_count(&objectCount, mempackBackend)) + throw GitError("querying the number of objects in a git memory packer backend"); + + if (!objectCount) + /* Nothing to do. */ + return; + checkInterrupt(); - git_buf buf = GIT_BUF_INIT; - Finally _disposeBuf{[&] { git_buf_dispose(&buf); }}; PackBuilder packBuilder; PackBuilderContext packBuilderContext; - git_packbuilder_new(Setter(packBuilder), *this); - git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext); + if (git_packbuilder_new(Setter(packBuilder), *this)) + throw GitError("creating git pack builder"); + + if (git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext)) + throw GitError("setting git pack builder callbacks"); + git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */); packBuilderContext.handleException( "preparing packfile", git_mempack_write_thin_pack(mempackBackend, packBuilder.get())); checkInterrupt(); - packBuilderContext.handleException("writing packfile", git_packbuilder_write_buf(&buf, packBuilder.get())); - checkInterrupt(); - std::string repo_path = std::string(git_repository_path(repo.get())); - while (!repo_path.empty() && repo_path.back() == '/') - repo_path.pop_back(); - std::string pack_dir_path = repo_path + "/objects/pack"; + auto packFilesPath = std::filesystem::path(git_repository_path(repo.get())) / "objects/pack"; - // TODO (performance): could the indexing be done in a separate thread? - // we'd need a more streaming variation of - // git_packbuilder_write_buf, or incur the cost of - // copying parts of the buffer to a separate thread. - // (synchronously on the git_packbuilder_write_buf thread) Indexer indexer; - git_indexer_progress stats; - if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) + if (git_indexer_new(Setter(indexer), packFilesPath.c_str(), 0, nullptr, nullptr)) throw GitError("creating git packfile indexer"); - // TODO: provide index callback for checkInterrupt() termination - // though this is about an order of magnitude faster than the packbuilder - // expect up to 1 sec latency due to uninterruptible git_indexer_append. - constexpr size_t chunkSize = 128 * 1024; - for (size_t offset = 0; offset < buf.size; offset += chunkSize) { - if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats)) - throw GitError("appending to git packfile index"); - checkInterrupt(); - } + struct State + { + Indexer & indexer; + PackBuilderContext & packBuilderContext; + GitIndexerSink sink{indexer.get()}; + }; + + State state{ + .indexer = indexer, + .packBuilderContext = packBuilderContext, + }; + + packBuilderContext.handleException( + "writing packfile", + git_packbuilder_foreach( + packBuilder.get(), + [](void * buf, size_t size, void * payload) -> int { + auto & state = *static_cast(payload); + try { + state.sink(std::string_view(static_cast(buf), size)); + } catch (...) { + state.packBuilderContext.exception = std::current_exception(); + return GIT_EUSER; + } + return GIT_OK; + }, + &state)); + + state.sink.flush(); + + auto & stats = state.sink.stats; if (git_indexer_commit(indexer.get(), &stats)) throw GitError("committing git packfile index"); @@ -380,6 +432,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_mempack_reset(mempackBackend)) throw GitError("resetting git mempack backend"); + debug( + "committed index and pack file to pack-%s.{idx,pack}, objects = %d, deltas = %d", + git_indexer_name(indexer.get()), + stats.total_objects, + stats.total_deltas); + checkInterrupt(); } @@ -389,7 +447,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this */ Pool getPool() { - // TODO: as an optimization, it would be nice to include `this` in the pool. return Pool(std::numeric_limits::max(), [this]() -> ref { auto repo = make_ref(path, options); @@ -570,7 +627,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile.string())) + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; @@ -782,8 +839,11 @@ ref GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti * Raw git tree input accessor. */ -struct GitSourceAccessor : SourceAccessor +struct GitSourceAccessor final : SourceAccessor { +private: + void anchor() override {}; +public: struct State { ref repo; @@ -812,20 +872,16 @@ struct GitSourceAccessor : SourceAccessor if (state->lfsFetch) { if (state->lfsFetch->shouldFetch(path)) { - StringSink s; try { // FIXME: do we need to hold the state lock while // doing this? auto contents = std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); - state->lfsFetch->fetch(contents, path, s, [&s](uint64_t size) { s.s.reserve(size); }); + state->lfsFetch->fetch(contents, path, sink, sizeCallback); } catch (Error & e) { e.addTrace({}, "while smudging git-lfs file '%s'", path); throw; } - sizeCallback(s.s.size()); - StringSource source{s.s}; - source.drainInto(sink); return; } } @@ -1060,8 +1116,11 @@ struct GitSourceAccessor : SourceAccessor } }; -struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor +struct GitExportIgnoreSourceAccessor final : CachingFilteringSourceAccessor { +private: + void anchor() override {}; +public: ref repo; std::optional rev; @@ -1120,7 +1179,11 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor } }; -struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink +void GitFileSystemObjectSink::anchor() {} + +namespace { + +struct GitFileSystemObjectSinkImpl final : GitFileSystemObjectSink { ref repo; @@ -1130,6 +1193,13 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink ThreadPool workers{concurrency}; + /** + * If repo has a non-null packBackend, this has a copy of the refresh function + * from the backend virtual table. This is needed to restore it after we've flushed + * the sink. We modify it to avoid unnecessary I/O on non-existent oids. + */ + decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr; + /** Total file contents in flight. */ std::atomic totalBufSize{0}; @@ -1139,13 +1209,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink : repo(repo) , repoPool(repo->getPool()) { + if (auto * backend = repo->packBackend) + packfileOdbRefresh = std::exchange(backend->refresh, nullptr); } + GitFileSystemObjectSinkImpl(GitFileSystemObjectSinkImpl &&) = delete; + GitFileSystemObjectSinkImpl(const GitFileSystemObjectSinkImpl &) = delete; + GitFileSystemObjectSinkImpl & operator=(GitFileSystemObjectSinkImpl &&) = delete; + GitFileSystemObjectSinkImpl & operator=(const GitFileSystemObjectSinkImpl &) = delete; + ~GitFileSystemObjectSinkImpl() { // Make sure the worker threads are destroyed before any state // they're referring to. workers.shutdown(); + if (auto * backend = repo->packBackend; backend && packfileOdbRefresh) + backend->refresh = packfileOdbRefresh; } struct Child; @@ -1153,7 +1232,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink /// A directory to be written as a Git tree. struct Directory { - std::map children; + std::map> children; std::optional oid; Child & lookup(const CanonPath & path) @@ -1162,7 +1241,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink auto parent = path.parent(); auto cur = this; for (auto & name : *parent) { - auto i = cur->children.find(std::string(name)); + auto i = cur->children.find(name); if (i == cur->children.end()) throw Error("path '%s' does not exist", path); auto dir = std::get_if(&i->second.file); @@ -1171,51 +1250,78 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink cur = dir; } - auto i = cur->children.find(std::string(*path.baseName())); + auto i = cur->children.find(*path.baseName()); if (i == cur->children.end()) throw Error("path '%s' does not exist", path); return i->second; } }; - size_t nextId = 0; // for Child.id + /* FIXME: Most of this logic is independent from git. Come up with a tree sink interface + and an adapter for a ExtendedFileSystemObjectSink that implements the tarball unpacking + semantics (i.e. overwriting of entries). Also deduplicate with MemorySourceAccessor. */ struct Child { git_filemode_t mode; - std::variant file; + std::variant> file; - /// Sequential numbering of the file in the tarball. This is - /// used to make sure we only import the latest version of a - /// path. - size_t id{0}; - }; - - struct State - { - Directory root; + const git_oid & getOid() const & + { + return std::visit( + overloaded{ + [](const Directory & dir) -> const git_oid & { return dir.oid.value(); }, + [](const git_oid & oid) -> const git_oid & { return oid; }, + [](const std::shared_future & oid) -> const git_oid & { return oid.get(); }, + }, + file); + } }; - Sync _state; + Directory root; - void addNode(State & state, const CanonPath & path, Child && child) + void addNode(const CanonPath & path, Child && child) { - assert(!path.isRoot()); + if (path.isRoot()) + throw Error("cannot create a file at the root of the git repository"); + auto parent = path.parent(); + assert(parent); - Directory * cur = &state.root; + Directory * cur = &root; for (auto & i : *parent) { auto child = std::get_if( &cur->children.emplace(std::string(i), Child{GIT_FILEMODE_TREE, {Directory()}}).first->second.file); - assert(child); + if (!child) + throw Error("parent of '%1%' is not a directory", path.rel()); cur = child; } std::string name(*path.baseName()); + auto prev = cur->children.find(name); + + if (prev == cur->children.end()) { + cur->children.insert_or_assign(std::move(name), std::move(child)); + return; + } + + /* Overwriting part of the tree. We'd like to behave somewhat + similarly to libarchive without ARCHIVE_EXTRACT_NO_OVERWRITE. */ + const auto & prevChild = prev->second; + + /* libarchive tries to unlink an entry, which only succeeds on empty + trees - so behave the same way. Everything else is fair game. */ + if (const auto * maybePrevDir = std::get_if(&prevChild.file)) { + /* "Replacing" directory with a directory is always a-ok. */ + if (std::holds_alternative(child.file)) + return; + + if (!maybePrevDir->children.empty()) + throw Error("cannot create '%1%', conflicting non-empty directory", path.rel()); + } - if (auto prev = cur->children.find(name); prev == cur->children.end() || prev->second.id < child.id) - cur->children.insert_or_assign(name, std::move(child)); + cur->children.insert_or_assign(std::move(name), std::move(child)); } void createRegularFile(const CanonPath & path, fun func) override @@ -1286,8 +1392,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink func(*crf); - auto id = nextId++; - if (crf->stream) { /* Finish the slow path by creating the blob object synchronously. Call .release(), since git_blob_create_from_stream_commit @@ -1295,48 +1399,52 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_oid oid; if (git_blob_create_from_stream_commit(&oid, crf->stream.release())) throw GitError("creating a blob object for '%s'", path); - addNode( - *_state.lock(), - crf->path, - Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id}); + addNode(crf->path, Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid}); return; } - /* Fast path: create the blob object in a separate thread. */ - workers.enqueue([this, crf{std::move(crf)}, id]() { - auto repo(repoPool.get()); - - git_oid oid; - if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size())) - throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path); - - addNode( - *_state.lock(), - crf->path, - Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id}); - }); + std::promise promise; + addNode( + crf->path, + Child{ + crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, + promise.get_future(), + }); + + /* Fast path: create the blob object in a separate thread. + FIXME: Ugly, make ThreadPool use std::move_only_function. */ + workers.enqueue( + [this, crf{std::move(crf)}, promise = make_ref(std::move(promise))]() mutable { + auto repo(repoPool.get()); + + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size())) + throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path); + + /* We don't generally bother with exceptions because those will + be propagated by the thread pool during .process(). */ + promise->set_value(oid); + }); } void createDirectory(const CanonPath & path) override { if (path.isRoot()) return; - auto state(_state.lock()); - addNode(*state, path, {GIT_FILEMODE_TREE, Directory()}); + addNode(path, {GIT_FILEMODE_TREE, Directory()}); } void createSymlink(const CanonPath & path, const std::string & target) override { - workers.enqueue([this, path, target]() { - auto repo(repoPool.get()); - - git_oid oid; - if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) - throw GitError("creating a blob object for tarball symlink member '%s'", path); - - auto state(_state.lock()); - addNode(*state, path, Child{GIT_FILEMODE_LINK, oid}); - }); + /* Symlinks are written to the this repo instance, the mempack backend + for which includes the trees. This way we flush both symlinks and + trees to the same packfile. Doing this synchronously isn't expensive + because symlinks are tiny, so hashing them is cheap. */ + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, requireCString(target), target.size())) + throw GitError("creating a blob object for tarball symlink member '%s'", path); + + addNode(path, Child{GIT_FILEMODE_LINK, oid}); } std::map hardLinks; @@ -1352,16 +1460,14 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink /* Create hard links. */ { - auto state(_state.lock()); for (auto & [path, target] : hardLinks) { if (target.isRoot()) continue; try { - auto child = state->root.lookup(target); - auto oid = std::get_if(&child.file); - if (!oid) + const auto & child = root.lookup(target); + if (std::holds_alternative(child.file)) throw Error("cannot create a hard link to a directory"); - addNode(*state, path, {child.mode, *oid}); + addNode(path, {child.mode, child.getOid()}); } catch (Error & e) { e.addTrace(nullptr, "while creating a hard link from '%s' to '%s'", path, target); throw; @@ -1375,30 +1481,34 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink ThreadPool workers{repos.size()}; for (auto & repo : repos) workers.enqueue([repo]() { repo->flush(); }); + workers.enqueue([repo = repo]() { repo->flush(); }); workers.process(); } + if (auto * backend = repo->packBackend) + /* We are done writing blobs. Need to refresh to get the objects written by other threads. */ + packfileOdbRefresh(backend); + // Write the Git trees to disk. Would be nice to have this multithreaded too, but that's hard because a tree // can't refer to an object that hasn't been written yet. Also it doesn't make a big difference for performance. - auto repo(repoPool.get()); - [&](this const auto & visit, Directory & node) -> void { + [&, &repo = *repo](this const auto & visit, Directory & node) -> void { checkInterrupt(); // Write the child directories. for (auto & child : node.children) if (auto dir = std::get_if(&child.second.file)) + /* TODO: Limit recursion depth? */ visit(*dir); // Write this directory. git_treebuilder * b; - if (git_treebuilder_new(&b, *repo, nullptr)) + if (git_treebuilder_new(&b, repo, nullptr)) throw GitError("creating a tree builder"); TreeBuilder builder(b); - for (auto & [name, child] : node.children) { - auto oid_p = std::get_if(&child.file); - auto oid = oid_p ? *oid_p : std::get(child.file).oid.value(); + for (const auto & [name, child] : node.children) { + const auto & oid = child.getOid(); if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode)) throw GitError("adding a file to a tree builder"); } @@ -1407,14 +1517,19 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink if (git_treebuilder_write(&oid, builder.get())) throw GitError("creating a tree object"); node.oid = oid; - }(_state.lock()->root); + }(root); repo->flush(); - return toHash(_state.lock()->root.oid.value()); + if (auto * backend = repo->packBackend) + backend->refresh = std::exchange(packfileOdbRefresh, nullptr); + + return toHash(root.oid.value()); } }; +} // namespace + ref GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options) { auto self = ref(shared_from_this()); @@ -1437,13 +1552,15 @@ ref GitRepoImpl::getAccessor( const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); - ref fileAccessor = AllowListSourceAccessor::create( - makeFSSourceAccessor(path), - /*allowedPrefixes=*/wd.files, - // Always allow access to the root, but not its children. - /*allowedPaths=*/{CanonPath::root}, - std::move(makeNotAllowedError)) - .cast(); + ref fileAccessor = + AllowListSourceAccessor::create( + // Follow the final symlink to the repo. Older nix versions used to do this (maybe somewhat accidentally). + makeFSSourceAccessor(path, /*trackLastModified=*/false, FinalSymlink::Follow), + /*allowedPrefixes=*/wd.files, + // Always allow access to the root, but not its children. + /*allowedPaths=*/{CanonPath::root}, + std::move(makeNotAllowedError)) + .cast(); if (options.exportIgnore) fileAccessor = make_ref(self, fileAccessor, std::nullopt); return fileAccessor; @@ -1467,6 +1584,7 @@ std::vector> GitRepoImpl::getSubmodules auto configS = accessor->readFile(modulesFile); auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + AutoDelete delTemp(pathTemp, /*recursive=*/false); try { writeFull(fdTemp.get(), configS); } catch (SystemError & e) { @@ -1485,6 +1603,7 @@ std::vector> GitRepoImpl::getSubmodules result.push_back({std::move(submodule), *rev}); } + delTemp.deletePath(); return result; } @@ -1502,9 +1621,11 @@ ref Settings::getTarballCache() const } // namespace fetchers +static Sync> workdirInfoCache_; + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + auto & _cache = workdirInfoCache_; { auto cache(_cache.lock()); auto i = cache->find(path); @@ -1516,6 +1637,11 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & return workdirInfo; } +void GitRepo::invalidateWorkdirInfoCache() +{ + workdirInfoCache_.lock()->clear(); +} + bool isLegalRefName(const std::string & refName) { initLibGit2(); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3941c3425660..f0d19945be10 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -13,6 +13,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/memo.hh" #include "nix/util/mounted-source-accessor.hh" #include @@ -21,8 +22,6 @@ # include #endif -using namespace std::string_literals; - namespace nix::fetchers { namespace { @@ -163,6 +162,13 @@ std::vector getPublicKeys(const Attrs & attrs) static const Hash nullRev{HashAlgorithm::SHA1}; +static LazyAttr makeLazyAttr(fun compute) +{ + return make_ref(LazyAttrComputation{ + .compute = memo(std::move(compute)), + }); +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override @@ -723,14 +729,12 @@ struct GitInputScheme : InputScheme } uint64_t getRevCount( - const Settings & settings, - const RepoInfo & repoInfo, - const std::filesystem::path & repoDir, - const Hash & rev) const + ref cache, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { - Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + if (GitRepo::openRepo(repoDir, {})->isShallow()) + throw Error("'%s' is a shallow Git repository, so 'revCount' is not available", repoInfo.locationToArg()); - auto cache = settings.getCache(); + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -745,6 +749,18 @@ struct GitInputScheme : InputScheme return revCount; } + LazyAttr lazyRevCount( + const Settings & settings, + const RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev) const + { + auto cache = settings.getCache(); + return makeLazyAttr([this, cache, repoInfo, repoDir, rev]() -> ResolvedAttr { + return getRevCount(cache, repoInfo, repoDir, rev); + }); + } + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( @@ -768,7 +784,7 @@ struct GitInputScheme : InputScheme "\n" "To make it visible to Nix, run:\n" "\n" - "git -C %2% add \"%1%\"", + "git -C %2% add -N \"%1%\"", path.rel(), PathFmt(repoPath)); else @@ -815,7 +831,7 @@ struct GitInputScheme : InputScheme repoDir = cacheDir; repoInfo.gitDir = "."; - std::filesystem::create_directories(cacheDir.parent_path()); + createDirs(cacheDir.parent_path()); PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true}); @@ -891,13 +907,6 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(repoDir, {}); - auto isShallow = repo->isShallow(); - - if (isShallow && !getShallowAttr(input)) - throw Error( - "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", - repoInfo.locationToArg()); - // FIXME: check whether rev is an ancestor of ref? auto rev = *input.getRev(); @@ -911,7 +920,7 @@ struct GitInputScheme : InputScheme if (!getShallowAttr(input)) { /* Like lastModified, skip revCount if supplied by the caller. */ if (!input.attrs.contains("revCount")) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoDir, rev)); } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -1033,8 +1042,11 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); if (!getShallowAttr(input)) { - input.attrs.insert_or_assign( - "revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); + if (rev == nullRev) { + input.attrs.insert_or_assign("revCount", uint64_t(0)); + } else { + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoPath, rev)); + } } verifyCommit(input, repo); @@ -1098,7 +1110,7 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath((*repoPath / file.rel()).string(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 8a21b8ddbf69..8eede58086e5 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/hash.hh" +#include "nix/util/ref.hh" +#include "nix/util/fun.hh" #include @@ -12,7 +14,24 @@ namespace nix::fetchers { -typedef std::variant> Attr; +/** + * The resolved (non-lazy) subset of attribute value types. + */ +using ResolvedAttr = std::variant>; + +/** + * A deferred attribute computation. Wrapping in `ref<>` gives + * pointer-identity equality/ordering, which is correct: two lazy + * attrs are equal iff they are the same computation. + */ +struct LazyAttrComputation +{ + fun compute; +}; + +using LazyAttr = ref; + +using Attr = std::variant, LazyAttr>; /** * An `Attrs` can be thought of a JSON object restricted or simplified @@ -21,6 +40,16 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * Force a potentially lazy attribute to its resolved value. + */ +ResolvedAttr forceAttr(const Attr & attr); + +/** + * Retrieve an attr, but only if it's a LazyAttr. + */ +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 7219635ec07d..df2aa8b7de4d 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -12,6 +12,11 @@ namespace nix::fetchers { */ struct Cache { +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); +public: virtual ~Cache() {} /** diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 2ab215a685ff..bb9a67d068d4 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -14,8 +14,9 @@ namespace nix { struct GitRepo; +struct SrcToStore; -} +} // namespace nix namespace nix::fetchers { @@ -152,7 +153,18 @@ struct Settings : public Config ref getTarballCache() const; + /** + * In-memory cache for calls to fetchToStore(); maps source paths to their store + * paths / hashes. + */ + static ref createSrcToStore(); + + const ref srcToStore = createSrcToStore(); + + private: + void anchor() override; + mutable Sync> _cache; }; diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 180d10e9dbbf..f65bbac73d4f 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -296,4 +296,4 @@ std::string publicKeys_to_string(const std::vector &); } // namespace nix::fetchers -JSON_IMPL(fetchers::PublicKey) +JSON_IMPL(nix::fetchers::PublicKey) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 13272719fe3d..c5c1ce282b30 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,7 +53,10 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; + void invalidateCache() override + { + next->invalidateCache(); + } /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` diff --git a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh index b59da391a056..444e0a648f14 100644 --- a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh +++ b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh @@ -38,9 +38,17 @@ struct Fetch void fetch( const std::string & content, const CanonPath & pointerFilePath, - StringSink & sink, + Sink & sink, std::function sizeCallback) const; std::vector fetchUrls(const std::vector & pointers) const; }; +struct LfsApiInfo +{ + std::string endpoint; + std::optional authHeader; +}; + +LfsApiInfo getLfsApi(ParsedURL url); + } // namespace nix::lfs diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 725fbf398410..f34f3bfde1b9 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -16,6 +16,10 @@ struct Settings; */ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { +private: + void anchor() override; + +public: /** * Flush builder and return a final Git hash. */ @@ -88,6 +92,9 @@ struct GitRepo static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Drop all entries from the getCachedWorkdirInfo() cache. */ + static void invalidateWorkdirInfoCache(); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 85a611355e2f..3fe96d8503bc 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" @@ -65,6 +66,10 @@ struct InputCacheImpl : InputCache void clear() override { cache_.lock()->clear(); + /* The workdir info cache has the same "per evaluation" lifetime + as the input cache, so flush it here as well so that e.g. + `:reload` in `nix repl` picks up changes in git work trees. */ + GitRepo::invalidateWorkdirInfoCache(); } }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index af5c94c1b494..6a4d239a03cf 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -14,8 +14,6 @@ #include #include -using namespace std::string_literals; - namespace nix::fetchers { static RunOptions hgOptions(OsStrings args) @@ -227,6 +225,8 @@ struct MercurialInputScheme : InputScheme input.attrs.insert_or_assign("ref", chomp(runHg({OS_STR("branch"), OS_STR("-R"), localPath.native()}))); + using namespace std::string_literals; + auto files = tokenizeString( runHg({ OS_STR("status"), @@ -240,26 +240,25 @@ struct MercurialInputScheme : InputScheme }), "\0"s); - auto actualPath = absPath(localPath); + /* FIXME: Check that the access to this path is allowed. */ + auto accessor = makeFSSourceAccessor(absPath(localPath)); PathFilter filter = [&](const std::string & p) -> bool { - assert(hasPrefix(p, actualPath.string())); - std::string file(p, actualPath.string().size() + 1); - - auto st = lstat(p); + auto cp = CanonPath(p); + auto st = accessor->lstat(cp); - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; + if (st.type == SourceAccessor::tDirectory) { + auto prefix = cp.rel() + "/"; auto i = files.lower_bound(prefix); return i != files.end() && hasPrefix(*i, prefix); } - return files.count(file); + return files.count(cp.rel()); }; return store.addToStore( input.getName(), - {getFSSourceAccessor(), CanonPath(actualPath.string())}, + {accessor, CanonPath::root}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, @@ -381,7 +380,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir / ".hg_archival.txt"); - auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root}); Attrs infoAttrs({ {"revCount", (uint64_t) revCount}, diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index d34dd4f434d1..ed52e565279d 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index fddb39bdf96b..4fef3ea0018a 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,16 +50,33 @@ headers += files('nix_api_flake.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixflakec', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'soversion' : nix_soversion, + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixflakec', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixflakec', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libflake-c/meson.options b/src/libflake-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libflake-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 8c6883d9cf95..e64179c72d1e 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 3512be10bce9..00b592195c2c 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 3ad66611726b..5ee587c3e051 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -19,10 +19,12 @@ #include "nix/fetchers/fetchers.hh" #include "nix/util/error.hh" #include "nix/util/experimental-features.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" +#include "nix/store/store-api.hh" namespace nix::flake::primops { @@ -42,7 +44,6 @@ PrimOp getFlake(const Settings & settings) auto path = state.realisePath(pos, *args[0]); callFlake(state, lockFlake(settings, state, path, lockFlags), v); } else { - NixStringContext context; std::string flakeRefS( state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); @@ -53,6 +54,21 @@ PrimOp getFlake(const Settings & settings) flakeRefS, state.positions[pos]); + /* Backwards compatibility: since flakes used to be copied to the store eagerly, some users + relied on being able to do builtins.getFlake on a flakeref with discarded string context. + So if a flake input has a physical source path that is inside the store, first try to look it up in the + storeFS. */ + if (auto sourcePath = flakeRef.input.getSourcePath(); + flakeRef.input.getType() == "path" && sourcePath && state.store->isInStore(sourcePath->string())) { + auto [storePath, subPath] = state.store->toStorePath(sourcePath->string()); + if (auto mount = state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + auto path = state.storePath(storePath) / CanonPath(subPath); + if (!flakeRef.subdir.empty()) + path = path / flakeRef.subdir; + return callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } + } + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); } }; @@ -89,12 +105,13 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); auto & vv = binds.alloc(s); + auto resolved = forceAttr(value); std::visit( overloaded{ [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const Explicit & value) { vv.mkBool(value.t); }}, - value); + resolved); } v.mkAttrs(binds); } diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index deb1c16b71e9..4f22353c751a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -38,6 +38,7 @@ #include "nix/fetchers/input-cache.hh" #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" @@ -391,14 +392,9 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } + auto rootDir = state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)); // Re-parse flake.nix from the store. - return readFlake( - state, - originalRef, - resolvedRef, - lockedRef, - state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)), - lockRootAttrPath); + return readFlake(state, originalRef, resolvedRef, lockedRef, rootDir, lockRootAttrPath); } Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) @@ -868,8 +864,6 @@ LockedFlake lockFlake( CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); - - flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level @@ -922,8 +916,9 @@ static ref makeInternalFS() internalFS->setPathDisplay("«flakes-internal»", ""); internalFS->addFile( CanonPath("call-flake.nix"), -#include "call-flake.nix.gen.hh" // IWYU pragma: keep - ); + { +#embed "call-flake.nix" + }); return internalFS; } @@ -995,11 +990,24 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); - /* Include revCount and lastModified because they're not - necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. */ - if (auto revCount = flake.lockedRef.input.getRevCount()) - *fingerprint += fmt(";revCount=%d", *revCount); + if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { + if (std::get_if(revCount)) { + /* A lazy revCount is computed by the fetcher, so its + value is functionally determined by `rev`. We only + need to record its presence, not force its value. + + This means a lazy and a concrete revCount that would + resolve to the same value produce different + fingerprints, sacrificing some cache hits to avoid + the cost of forcing. */ + *fingerprint += ";hasRevCount"; + } else if (auto n = flake.lockedRef.input.getRevCount()) { + /* A concrete revCount comes from a lockfile or explicit + user input. The fetcher passes it through as-is, so + it can affect evaluation and must be fingerprinted. */ + *fingerprint += fmt(";revCount=%d", *n); + } + } if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fd52dbebac5d..aa063a08e396 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -233,14 +233,6 @@ ref openEvalCache(EvalState & state, ref= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -32,13 +32,6 @@ subdir('nix-meson-build-support/common') subdir('nix-meson-build-support/generate-header') -generated_headers = [] -foreach header : [ - 'call-flake.nix', -] - generated_headers += gen_header.process(header) -endforeach - sources = files( 'config.cc', 'flake-primops.cc', @@ -57,7 +50,6 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixflake', sources, - generated_headers, soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 36332fdb70a1..998151142fda 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -42,16 +42,33 @@ headers = files( subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixmainc', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'soversion' : nix_soversion, + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixmainc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixmainc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libmain-c/meson.options b/src/libmain-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libmain-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index f019a917d360..5f86df5ea577 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c47..b20df3a99ec9 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -81,8 +81,8 @@ struct MixPrintJSON : virtual Args * This is a template to avoid accidental coercions from `string` to `json` in the caller, * to avoid mistakenly passing an already serialized JSON to this function. * - * It is not recommended to print a JSON string - see the JSON guidelines - * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * It is not recommended to print a JSON string - see the data modeling guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/data-modeling.html - * but you _can_ print a sole JSON string by explicitly coercing it to * `nlohmann::json` first. */ diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index f9e771205ba2..7b0e7a54d4e0 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -26,7 +26,7 @@ void parseCmdLine( const Strings & args, fun parseArg); -void printVersion(const std::string & programName); +[[noreturn]] void printVersion(const std::string & programName); /** * Ugh. No better place to put this. diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index a3e75c535dd0..a1c3768f2b15 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -50,7 +50,8 @@ void setLogFormat(const std::string & logFormatStr) void setLogFormat(const LogFormat & logFormat) { defaultLogFormat = logFormat; - logger = makeDefaultLogger(); + logger->stop(); + logger = makeDefaultLogger().release(); } } // namespace nix diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 2ac59924e592..0084643bdaad 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a8044a240d51..512ba4faad67 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -2,8 +2,10 @@ #include "nix/util/terminal.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/path.hh" +#include "nix/util/file-system.hh" #include "nix/store/names.hh" +#include "nix/util/util.hh" #include #include @@ -13,17 +15,19 @@ namespace nix { +namespace { + static std::string_view getS(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + throw Error("could not get expected log field of type 'string' at index %d", n); return fields[n].s; } static uint64_t getI(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + throw Error("could not get expected log field of type 'int' at index %d", n); return fields[n].i; } @@ -34,10 +38,17 @@ static std::string_view storePathToName(std::string_view path) return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); } -class ProgressBar : public Logger +static std::string_view storePathToNameWithoutDrvSuffix(std::string_view path) { -private: + auto res = storePathToName(path); + if (hasSuffix(res, drvExtension)) + res.remove_suffix(drvExtension.size()); + return res; +} +class ProgressBar final : public Logger +{ +private: struct ActInfo { std::string s, lastLine, phase; @@ -96,6 +107,18 @@ class ProgressBar : public Logger std::unique_ptr interruptCallback; + void hideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25l"); + } + + void unhideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25h"); + } + public: ProgressBar(bool isTTY) @@ -105,6 +128,7 @@ class ProgressBar : public Logger redraw("\rshutting down\e[K"); })) { + hideCursorIfNeeded(); state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); @@ -131,6 +155,7 @@ class ProgressBar : public Logger if (state->active) { state->active = false; clearProgressDisplay(); + unhideCursorIfNeeded(); updateCV.notify_one(); quitCV.notify_one(); } @@ -148,8 +173,10 @@ class ProgressBar : public Logger return; } - if (state->active) + if (state->active) { clearProgressDisplay(); + unhideCursorIfNeeded(); + } } void resume() override @@ -162,8 +189,10 @@ class ProgressBar : public Logger state->suspensions--; } if (state->suspensions == 0) { - if (state->active) + if (state->active) { clearProgressDisplay(); + hideCursorIfNeeded(); + } state->haveUpdate = true; updateCV.notify_one(); } @@ -223,9 +252,7 @@ class ProgressBar : public Logger state->activitiesByType[type].its.emplace(act, i); if (type == actBuild) { - std::string name(storePathToName(getS(fields, 0))); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") @@ -250,9 +277,7 @@ class ProgressBar : public Logger } if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } @@ -674,7 +699,9 @@ class ProgressBar : public Logger return {}; invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); + unhideCursorIfNeeded(); auto s = trim(readLine(getStandardInput(), true)); + hideCursorIfNeeded(); if (s.size() != 1) return {}; draw(*state); @@ -687,6 +714,8 @@ class ProgressBar : public Logger } }; +} // namespace + std::unique_ptr makeProgressBar() { return std::make_unique(isTTY()); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index c57f71cc2689..f1d419af5ed1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -66,8 +66,7 @@ void printMissing(ref store, const MissingPaths & missing, Verbosity lvl) else printMsg(lvl, "these %d derivations will be built:", missing.willBuild.size()); auto sorted = store->topoSortPaths(missing.willBuild); - reverse(sorted.begin(), sorted.end()); - for (auto & i : sorted) + for (auto & i : sorted | std::views::reverse) printMsg(lvl, " %s", store->printStorePath(i)); } diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index c81235bf16d4..600de4d2ea61 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -45,16 +45,33 @@ headers += files('nix_api_store_internal.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixstorec', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'soversion' : nix_soversion, + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixstorec', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixstorec', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libstore-c/meson.options b/src/libstore-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libstore-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fbb3c418566c..bce1cdd22b22 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -8,6 +8,7 @@ #include "nix/store/path.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/store-open.hh" #include "nix/store/store-reference.hh" #include "nix/store/build-result.hh" @@ -178,7 +179,7 @@ nix_err nix_store_realise( .drvPath = nix::makeConstantStorePathRef(path->path), .outputs = nix::OutputsSpec::All{}}}; const auto nixStore = store->ptr; - auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore); + auto results = nixStore->getBuilder(nixStore)->buildPathsWithResults(paths, nix::bmNormal); assert(results.size() == 1); diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fde17c78e017..822acb00fc70 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -8,6 +8,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -25,7 +26,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -37,6 +38,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index ee8018c3268f..8440a74df912 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,16 +1,13 @@ - -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::map(gen::arbitrary(), [](StorePath path) { return DerivedPath::Opaque{ .path = path, @@ -18,8 +15,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::arbitrary(), [](SingleDerivedPath drvPath) { return gen::map(gen::arbitrary(), [drvPath](StorePathName outputPath) { return SingleDerivedPath::Built{ @@ -30,8 +28,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::arbitrary(), [](SingleDerivedPath drvPath) { return gen::map(gen::arbitrary(), [drvPath](OutputsSpec outputs) { return DerivedPath::Built{ @@ -42,8 +41,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::inRange(0, std::variant_size_v), [](uint8_t n) { switch (n) { case 0: @@ -56,8 +56,9 @@ Gen Arbitrary::arbitrary() }); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using namespace nix; return gen::mapcat(gen::inRange(0, std::variant_size_v), [](uint8_t n) { switch (n) { case 0: diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index 15df329cb2b1..df48a7469ea8 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -22,13 +22,10 @@ public: }; ~nix_api_store_test_base() override - { - if (exists(std::filesystem::path{nixDir})) { - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); - } - std::filesystem::remove_all(nixDir); - } + try { + nix::deletePath(nixDir); + } catch (...) { + nix::ignoreExceptionInDestructor(); } std::string nixDir; diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index a30f83770257..6cdb0a60ef96 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/outputs-spec.hh" diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 4d904cb1d06a..ca451db4abce 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 98a255ccc026..ce14f469c7f1 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include @@ -57,7 +55,7 @@ Gen Arbitrary::arbitrary() })); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { return gen::construct( gen::arbitrary(), diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index a1d8ddee6412..9215b99e041f 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -36,8 +36,6 @@ TEST_P(BuildResultJsonTest, to_json) writeJsonTest(name, value); } -using namespace std::literals::chrono_literals; - INSTANTIATE_TEST_SUITE_P( BuildResultJSON, BuildResultJsonTest, @@ -89,8 +87,8 @@ INSTANTIATE_TEST_SUITE_P( .timesBuilt = 3, .startTime = 30, .stopTime = 50, - .cpuUser = std::chrono::microseconds(500s), - .cpuSystem = std::chrono::microseconds(604s), + .cpuUser = std::chrono::seconds(500), + .cpuSystem = std::chrono::seconds(604), }, })); diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index bb7ad28f3d78..5a20c4f91539 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -12,8 +12,6 @@ namespace nix { -using namespace nlohmann; - class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, public LibStoreTest { protected: @@ -39,7 +37,7 @@ class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, this->readTest(fileName, [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures); }); } @@ -55,7 +53,7 @@ class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, this->readTest(fileName, [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_EQ(options, expected); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures); @@ -83,6 +81,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); #define TEST_ATERM_JSON(STEM, NAME) \ TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ { \ + using namespace nlohmann; \ this->readTest(NAME ".json", [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ /* Use DRV file instead of C++ literal as source of truth. */ \ @@ -95,6 +94,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); \ TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ { \ + using namespace nlohmann; \ this->writeTest( \ NAME ".json", \ [&]() -> json { \ @@ -108,6 +108,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); \ TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ { \ + using namespace nlohmann; \ this->readTest(NAME ".drv", [&](auto encoded) { \ /* Use JSON file instead of C++ literal as source of truth. */ \ auto j = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ @@ -184,7 +185,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(!got.structuredAttrs); @@ -228,7 +229,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(!got.structuredAttrs); @@ -324,7 +325,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(got.structuredAttrs); @@ -373,7 +374,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto options = derivationOptionsFromStructuredAttrs( - *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); + *this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings); EXPECT_TRUE(got.structuredAttrs); diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 6fee675e9849..412d192ced62 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -15,6 +15,14 @@ TEST_F(DerivationTest, BadATerm_version) parseDerivation(*store, readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); } +TEST_F(DerivationTest, UnterminatedString) +{ + ASSERT_THROW( + parseDerivation( + *store, "Derive([(\"out\",\"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo", "bar", mockXpSettings), + FormatError); +} + TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( @@ -66,11 +74,10 @@ INSTANTIATE_TEST_SUITE_P( std::pair{ "caFixedNAR", DerivationOutput{DerivationOutput::CAFixed{ - .ca = - { - .method = ContentAddressMethod::Raw::NixArchive, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, + .ca{ + .method = ContentAddressMethod::Raw::NixArchive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, }}, }, std::pair{ @@ -177,41 +184,36 @@ struct DerivationJsonAtermTest : DerivationTest, MAKE_TEST_P(DerivationJsonAtermTest); -INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = - { - { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = - { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; - }())); +INSTANTIATE_TEST_SUITE_P( + DerivationJSONATerm, + DerivationJsonAtermTest, + ::testing::Values( + Derivation{ + .outputs = {}, + .inputs{ + .srcs{ + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, + }, + .drvs{.map{ + { + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"}, + { + .value{ + "cat", + "dog", + }, + }, + }, + }}, + }, + .platform = "wasm-sel4", + .builder = "foo", + .args = {"bar", "baz"}, + .env{ + {"BIG_BAD", "WOLF"}, + }, + .name = "simple-derivation", + })); struct DynDerivationJsonAtermTest : DynDerivationTest, JsonCharacterizationTest, @@ -222,60 +224,36 @@ MAKE_TEST_P(DynDerivationJsonAtermTest); Derivation makeDynDepDerivation() { - Derivation drv; - drv.name = "dyn-dep-derivation"; - drv.inputSrcs = { - StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, - }; - drv.inputDrvs = { - .map = - { + return Derivation{ + .outputs = {}, + .inputs{ + .srcs{ + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, + }, + .drvs{.map{ { StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"}, DerivedPathMap::ChildNode{ - .value = - { - "cat", - "dog", - }, - .childMap = - { - { - "cat", - DerivedPathMap::ChildNode{ - .value = - { - "kitten", - }, - }, - }, - { - "goose", - DerivedPathMap::ChildNode{ - .value = - { - "gosling", - }, - }, - }, - }, + .value{ + "cat", + "dog", + }, + .childMap{ + {"cat", DerivedPathMap::ChildNode{.value = {"kitten"}}}, + {"goose", DerivedPathMap::ChildNode{.value = {"gosling"}}}, + }, }, }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", + }}, + }, + .platform = "wasm-sel4", + .builder = "foo", + .args = {"bar", "baz"}, + .env{ + {"BIG_BAD", "WOLF"}, }, + .name = "dyn-dep-derivation", }; - return drv; } INSTANTIATE_TEST_SUITE_P(DynDerivationJSONATerm, DynDerivationJsonAtermTest, ::testing::Values(makeDynDepDerivation())); diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index e825655d66f4..eea26b78e99a 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -29,21 +29,21 @@ class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTe */ StorePath makeCAFloatingDependency(std::string_view name) { - Derivation depDrv; - depDrv.name = name; - depDrv.platform = "x86_64-linux"; - depDrv.builder = "/bin/sh"; - depDrv.outputs = { - { - "out", - // will ensure that downstream is deferred - DerivationOutput{DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }}, + Derivation depDrv{ + .outputs{ + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, }, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"out", ""}}, + .name = std::string{name}, }; - depDrv.env = {{"out", ""}}; // Fill in the dependency derivation's output paths depDrv.fillInOutputPaths(*store); @@ -64,14 +64,13 @@ TEST_F(FillInOutputPathsTest, fillsDeferredOutputs_emptyStringEnvVar) using nlohmann::json; // Before: Derivation with deferred output - Derivation drv; - drv.name = "filled-in-deferred-empty-env-var"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Fill in deferred output with empty env var"}, {"out", ""}}, + .name = "filled-in-deferred-empty-env-var", }; - drv.env = {{"__doc", "Fill in deferred output with empty env var"}, {"out", ""}}; // Serialize before state checkpointJson("filled-in-deferred-empty-env-var-pre", drv); @@ -95,15 +94,12 @@ TEST_F(FillInOutputPathsTest, fillsDeferredOutputs_empty_string_var) using nlohmann::json; // Before: Derivation with deferred output - Derivation drv; - drv.name = "filled-in-deferred-no-env-var"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, - }; - drv.env = { - {"__doc", "Fill in deferred with missing env var"}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Fill in deferred with missing env var"}}, + .name = "filled-in-deferred-no-env-var", }; // Serialize before state @@ -127,16 +123,12 @@ TEST_F(FillInOutputPathsTest, preservesInputAddressedOutputs) { auto expectedPath = StorePath{"w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"}; - Derivation drv; - drv.name = "filled-in-already"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::InputAddressed{.path = expectedPath}}}, - }; - drv.env = { - {"__doc", "Correct path stays unchanged"}, - {"out", store->printStorePath(expectedPath)}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = expectedPath}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Correct path stays unchanged"}, {"out", store->printStorePath(expectedPath)}}, + .name = "filled-in-already", }; // Serialize before state @@ -154,16 +146,12 @@ TEST_F(FillInOutputPathsTest, throwsOnIncorrectInputAddressedPath) { auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"}; - Derivation drv; - drv.name = "bad-path"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}, - }; - drv.env = { - {"__doc", "Wrong InputAddressed path throws error"}, - {"out", store->printStorePath(wrongPath)}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Wrong InputAddressed path throws error"}, {"out", store->printStorePath(wrongPath)}}, + .name = "bad-path", }; // Serialize before state @@ -177,16 +165,12 @@ TEST_F(FillInOutputPathsTest, throwsOnIncorrectEnvVar) { auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"}; - Derivation drv; - drv.name = "bad-env-var"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, - }; - drv.env = { - {"__doc", "Wrong env var value throws error"}, - {"out", store->printStorePath(wrongPath)}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Wrong env var value throws error"}, {"out", store->printStorePath(wrongPath)}}, + .name = "bad-env-var", }; // Serialize before state @@ -204,19 +188,14 @@ TEST_F(FillInOutputPathsTest, preservesDeferredWithInputDrvs) auto depDrvPath = makeCAFloatingDependency("dependency"); // Create a derivation that depends on the dependency - Derivation drv; - drv.name = "depends-on-drv"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::Deferred{}}}, - }; - drv.env = { - {"__doc", "Deferred stays deferred with CA dependencies"}, - {"out", ""}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "Deferred stays deferred with CA dependencies"}, {"out", ""}}, + .name = "depends-on-drv", }; - // Add the real input derivation dependency - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Serialize before state checkpointJson("depends-on-drv-pre", drv); @@ -240,19 +219,14 @@ TEST_F(FillInOutputPathsTest, throwsOnPatWhenShouldBeDeffered) auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"}; // Create a derivation that depends on the dependency - Derivation drv; - drv.name = "depends-on-drv"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/sh"; - drv.outputs = { - {"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}, - }; - drv.env = { - {"__doc", "InputAddressed throws when should be deferred"}, - {"out", ""}, + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .env = {{"__doc", "InputAddressed throws when should be deferred"}, {"out", ""}}, + .name = "depends-on-drv", }; - // Add the real input derivation dependency - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Serialize before state checkpointJson("bad-depends-on-drv-pre", drv); diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc index 60b86f571205..03a67a8795f5 100644 --- a/src/libstore-tests/derivations.cc +++ b/src/libstore-tests/derivations.cc @@ -128,13 +128,13 @@ TEST_F(TryResolveTest, noInputs) resolveExpect( "no-inputs", [&] { - Derivation drv; - drv.name = "no-inputs"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.env = {{"FOO", "bar"}}; - return drv; + return Derivation{ + .outputs = {{"out", caFloatingOutput()}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .env = {{"FOO", "bar"}}, + .name = "no-inputs", + }; }(), {}, [&] { @@ -168,28 +168,28 @@ TEST_F(TryResolveTest, withInputs) resolveExpect( "with-inputs", - [&] { - Derivation drv; - drv.name = "with-inputs"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = multiOutputs; - drv.inputDrvs = { - .map = { + Derivation{ + .outputs = multiOutputs, + .inputs{ + .drvs{.map{ {dep1DrvPath, {.value = {"out", "dev"}}}, {dep2DrvPath, {.value = {"out"}}}, - }}; - drv.env = { - {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, - {"DEP1_DEV", placeholder1Dev}, - {"DEP2", placeholder2Out}, - }; - drv.structuredAttrs = StructuredAttrs{{ + }}, + }, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .env = + { + {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, + {"DEP1_DEV", placeholder1Dev}, + {"DEP2", placeholder2Out}, + }, + .structuredAttrs = StructuredAttrs{{ {"dep1out", placeholder1Out}, {"nested", nlohmann::json::object({{"dep2", "before " + placeholder2Out + " after"}})}, - }}; - return drv; - }(), + }}, + .name = "with-inputs", + }, {.dict{ { SingleDerivedPath::Built{ @@ -219,7 +219,7 @@ TEST_F(TryResolveTest, withInputs) expected.platform = "x86_64-linux"; expected.builder = "/bin/bash"; expected.outputs = multiOutputs; - expected.inputSrcs = {dep1OutPath, dep1DevPath, dep2OutPath}; + expected.inputs = {dep1OutPath, dep1DevPath, dep2OutPath}; expected.env = { {"DEP1_OUT", "prefix-" + store->printStorePath(dep1OutPath) + "-suffix"}, {"DEP1_DEV", store->printStorePath(dep1DevPath)}, @@ -238,12 +238,13 @@ TEST_F(TryResolveTest, resolutionFailure) { StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; - Derivation drv; - drv.name = "resolution-failure"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + Derivation drv{ + .outputs = {{"out", caFloatingOutput()}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .name = "resolution-failure", + }; BuildTrace buildTrace; @@ -273,7 +274,7 @@ void TryResolveTest::exportRefGraphSubpathTest( nix::checkpointJson(*this, std::string{stem} + "-before", drv); - auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputDrvs, drv.env, parsed, true); + auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputs.drvs, drv.env, parsed, true); nix::checkpointJson(*this, std::string{stem} + "-before-options", options); @@ -328,14 +329,13 @@ TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); - Derivation drv; - drv.name = "export-ref-subpath"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; - drv.env = { - {"exportReferencesGraph", "refs " + placeholder + "/foo"}, + Derivation drv{ + .outputs = {{"out", caFloatingOutput()}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .env = {{"exportReferencesGraph", "refs " + placeholder + "/foo"}}, + .name = "export-ref-subpath", }; exportRefGraphSubpathTest("export-ref-subpath", drv, nullptr); @@ -346,15 +346,20 @@ TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); - Derivation drv; - drv.name = "export-ref-subpath-sa"; - drv.platform = "x86_64-linux"; - drv.builder = "/bin/bash"; - drv.outputs = {{"out", caFloatingOutput()}}; - drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; - drv.structuredAttrs = StructuredAttrs{{ - {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, - }}; + Derivation drv{ + .outputs = {{"out", caFloatingOutput()}}, + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .platform = "x86_64-linux", + .builder = "/bin/bash", + .structuredAttrs = StructuredAttrs{{ + { + "exportReferencesGraph", + nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}}), + }, + }}, + .name = "export-ref-subpath-sa", + }; + // env depends on structuredAttrs, so set it after construction drv.env = { {std::string{StructuredAttrs::envVarName}, nlohmann::json(drv.structuredAttrs->structuredAttrs).dump()}, }; diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index 541d729e030c..46297c9a30db 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -92,8 +92,6 @@ TEST_F(DerivedPathTest, built_built_xp) MissingExperimentalFeature); } -#ifndef COVERAGE - /* TODO: Disabled due to the following error: path '00000000000000000000000000000000-0^0' is not a valid store path: @@ -113,8 +111,6 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o)) RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store), xpSettings)); } -#endif - /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 4626fc31f28d..6c2625d5306a 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -158,9 +158,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { "one-derivation", [&] { auto store = writeCfg->openDummyStore(); - Derivation drv; - drv.name = "foo"; - store->writeDerivation(drv); + store->writeDerivation(Derivation{.name = "foo"}); return store; }(), }, diff --git a/src/libstore-tests/filetransfer-request.cc b/src/libstore-tests/filetransfer-request.cc new file mode 100644 index 000000000000..b89bb7ed0bc1 --- /dev/null +++ b/src/libstore-tests/filetransfer-request.cc @@ -0,0 +1,19 @@ +#include + +#include "nix/store/filetransfer.hh" + +namespace nix { + +TEST(FileTransferRequest, displayUriStripsUserinfo) +{ + FileTransferRequest req(VerbatimURL{std::string{"https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"}}); + // uri itself is untouched (used for CURLOPT_URL, result.urls, cache keys). + EXPECT_EQ(req.uri.to_string(), "https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"); + // displayUri() drops the userinfo for diagnostics. + EXPECT_EQ(req.displayUri(), "https://example.org:8443/path/file.toml?x=1"); + + FileTransferRequest plain(VerbatimURL{std::string{"https://example.org/file"}}); + EXPECT_EQ(plain.displayUri(), "https://example.org/file"); +} + +} // namespace nix diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index 74f3b93cd3df..93d279d53a1f 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -57,13 +57,10 @@ TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams) using testing::HttpsBinaryCacheStoreMtlsTest; using testing::HttpsBinaryCacheStoreTest; -using namespace std::string_view_literals; -using namespace std::string_literals; - TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo) { auto store = openStore(makeConfig()); - StringSource dump{"test"sv}; + StringSource dump{std::string_view("test")}; auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); EXPECT_NO_THROW(store->queryPathInfo(path)); } @@ -74,7 +71,7 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo) config->tlsCert = clientCert; config->tlsKey = clientKey; auto store = openStore(config); - StringSource dump{"test"sv}; + StringSource dump{std::string_view("test")}; auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); EXPECT_NO_THROW(store->queryPathInfo(path)); } @@ -104,7 +101,7 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert) TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority) { - StringSource dump{"test"sv}; + StringSource dump{std::string_view("test")}; auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); for (auto & entry : DirectoryIterator{cacheDir}) diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index a126a87aca8b..a7353f1ec7dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -64,6 +64,7 @@ sources = files( 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-request.cc', 'filetransfer-retry.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', @@ -76,6 +77,7 @@ sources = files( 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', + 'outputs-query.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', diff --git a/src/libstore-tests/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc index aebefc775675..7612250c661d 100644 --- a/src/libstore-tests/nar-info-disk-cache.cc +++ b/src/libstore-tests/nar-info-disk-cache.cc @@ -30,15 +30,16 @@ TEST(NarInfoDiskCacheImpl, create_and_read) // Set up "background noise" and check that different caches receive different ids { - auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); - auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + auto bc1 = + cache->createCache("https://bar", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.priority = 12}); ASSERT_NE(bc1, bc2); barId = bc1; } // Check that the fields are saved and returned correctly. This does not test // the select statement yet, because of in-memory caching. - savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + savedId = cache->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); ; { auto r = cache->upToDateCacheExists("http://foo"); @@ -84,7 +85,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read) } // "Update", same data, check that the id number is reused - cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + cache2->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); { auto r = cache2->upToDateCacheExists("http://foo"); @@ -107,7 +108,8 @@ TEST(NarInfoDiskCacheImpl, create_and_read) auto r0 = cache2->upToDateCacheExists("https://bar"); ASSERT_FALSE(r0); - cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + cache2->createCache( + "https://bar", "/nix/storedir", {.wantMassQuery = !wantMassQuery, .priority = prio + 10}); auto r = cache2->upToDateCacheExists("https://bar"); ASSERT_EQ(r->wantMassQuery, !wantMassQuery); ASSERT_EQ(r->priority, prio + 10); diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 21a1d63a7e94..63e5908d8818 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -75,7 +75,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) }; info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; - info.compression = "xz"; + info.compression = CompressionAlgo::xz; info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); info.fileSize = 4029176; } diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 60626869a919..0162684daf4b 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -77,8 +77,8 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) { StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); ASSERT_NE(result, nullptr); - ASSERT_STREQ("name", result->path.name().data()); - ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); + ASSERT_EQ("name", result->path.name()); + ASSERT_EQ(PATH_SUFFIX.substr(1), result->path.to_string()); nix_store_path_free(result); } diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc new file mode 100644 index 000000000000..356df632b8da --- /dev/null +++ b/src/libstore-tests/outputs-query.cc @@ -0,0 +1,114 @@ +// Regression tests for the functions in outputs-query.cc +// +// See https://github.com/NixOS/nix/issues/15713 + +#include + +#include "nix/store/outputs-query.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class OutputsQueryTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + EnableExperimentalFeature caFeature{"ca-derivations"}; + + ref store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openDummyStore(); + }(); + + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a simple floating CA derivation with a given name and no input + * derivations. + */ + Derivation makeLeafDrv(std::string name) + { + return Derivation{ + .outputs = {{"out", caFloatingOutput()}}, + .platform = "x86_64-linux", + .builder = "/bin/sh", + .name = std::move(name), + }; + } +}; + +/** + * Regression test for https://github.com/NixOS/nix/issues/15713 + * + * In a Fibonacci-style chain of floating CA derivations, the resolution + * algorithm used to call queryRealisation O(Fib(N)) times. + * This test verifies that memoization reduces this to O(N). + */ +TEST_F(OutputsQueryTest, fibonacciChainQueryCount) +{ + constexpr static size_t N = 10; + std::vector drvPaths; + + // d0, d1: leaf derivations + for (int i = 0; i < 2; ++i) { + drvPaths.push_back(store->writeDerivation(makeLeafDrv("d" + std::to_string(i)))); + } + + // d_i depends on d_{i-1} and d_{i-2} + for (size_t i = 2; i <= N; ++i) { + Derivation drv = makeLeafDrv("d" + std::to_string(i)); + drv.inputs.drvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputs.drvs.map[drvPaths[i - 2]].value.insert("out"); + drvPaths.push_back(store->writeDerivation(drv)); + } + + // Tracker for queryRealisation calls. + std::map callCounts; + std::map outPaths; + + QueryRealisationFun queryRealisation = [&](const DrvOutput & id) -> std::shared_ptr { + assert(id.outputName == "out"); + callCounts[id.drvPath]++; + + // Memoize mock output paths. + auto it = outPaths.find(id.drvPath); + if (it == outPaths.end()) { + auto hash = hashString(HashAlgorithm::SHA1, "mock-output-" + std::to_string(outPaths.size())); + it = outPaths.emplace(id.drvPath, StorePath(hash, "out")).first; + } + + return std::make_shared(UnkeyedRealisation{.outPath = it->second}); + }; + + auto result = deepQueryPartialDerivationOutput(*store, drvPaths[N], "out", nullptr, queryRealisation); + + ASSERT_TRUE(result); + + int totalCalls = 0; + for (auto & [path, count] : callCounts) { + totalCalls += count; + if (count > 1) + ADD_FAILURE() << "Derivation at " << store->printStorePath(path) << " was queried " << count + << " times (expected 1)"; + } + + // With full memoization (ResolveCache + RealisationCache), each derivation should be queried exactly once. + EXPECT_EQ(totalCalls, N + 1) << "queryRealisation called " << totalCalls << " times; expected exactly " << (N + 1); +} + +} // namespace nix diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index d791709ac258..ae0bb1789b2b 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -268,13 +268,9 @@ INSTANTIATE_TEST_SUITE_P( #undef TEST_JSON -#ifndef COVERAGE - RC_GTEST_PROP(OutputsSpec, prop_round_rip, (const OutputsSpec & o)) { RC_ASSERT(o == OutputsSpec::parse(o.to_string())); } -#endif - } // namespace nix diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index eb860a34dab1..65755b7f9645 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -86,8 +86,6 @@ TEST_DO_PARSE(triple_dot, "...") #undef TEST_DO_PARSE -#ifndef COVERAGE - RC_GTEST_FIXTURE_PROP(StorePathTest, prop_regex_accept, (const StorePath & p)) { RC_ASSERT(std::regex_match(std::string{p.name()}, nameRegex)); @@ -141,8 +139,6 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) RC_ASSERT(parsed == std::regex_match(std::string{name}, nameRegex)); } -#endif - /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index f2c6fb51e5ca..69b89b1460ee 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -4,6 +4,9 @@ #include +#include +#include + namespace nix { struct RewriteParams @@ -43,6 +46,100 @@ INSTANTIATE_TEST_CASE_P( RewriteParams{"foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, RewriteParams{"foooo", "foooo", {}})); +TEST(references, rewritingSinkChunking) +{ + std::mt19937 rng(42); + + /* Build a set of rewrites. Keys are random [a-z] strings of random + length in [8, 32]; values are same-length [A-Z] strings so a + replacement never produces a new match for any key. We also skip + any key that would be a substring of (or have as a substring) an + existing key, so that inserting one key into the input cannot also + produce a match for a different key. */ + StringMap rewrites; + std::vector keys; + { + std::uniform_int_distribution lowerDist('a', 'z'); + std::uniform_int_distribution upperDist('A', 'Z'); + std::uniform_int_distribution lenDist(8, 32); + + while (rewrites.size() < 8) { + std::string from(lenDist(rng), '\0'); + for (auto & c : from) + c = lowerDist(rng); + if (rewrites.count(from)) + continue; + bool overlap = false; + for (auto & other : keys) + if (from.find(other) != std::string::npos || other.find(from) != std::string::npos) { + overlap = true; + break; + } + if (overlap) + continue; + std::string to(from.size(), '\0'); + for (auto & c : to) + c = upperDist(rng); + rewrites[from] = to; + keys.push_back(from); + } + } + + /* Build a ~1 MB input mixing rewrite keys with [0-9] digits. Always + emit at least one digit between two consecutive keys so adjacency + cannot create spurious matches at key boundaries. Compute the + expected output string and matches vector at the same time. */ + std::string input; + std::string expectedOutput; + std::set expectedMatches; + input.reserve(1'000'000); + expectedOutput.reserve(1'000'000); + { + std::uniform_int_distribution digitDist('0', '9'); + std::uniform_int_distribution keyDist(0, keys.size() - 1); + std::bernoulli_distribution useKeyDist(0.2); + bool justInsertedKey = false; + + while (input.size() < 1'000'000) { + if (useKeyDist(rng) && !justInsertedKey) { + const auto & from = keys[keyDist(rng)]; + expectedMatches.insert(input.size()); + expectedOutput += rewrites.at(from); + input += from; + justInsertedKey = true; + } else { + char d = static_cast(digitDist(rng)); + input.push_back(d); + expectedOutput.push_back(d); + justInsertedKey = false; + } + } + } + + StringSink singleOut; + RewritingSink singleSink(rewrites, singleOut); + singleSink(input); + singleSink.flush(); + + StringSink chunkedOut; + RewritingSink chunkedSink(rewrites, chunkedOut); + { + std::uniform_int_distribution chunkDist(1, 128); + std::string_view remaining(input); + while (!remaining.empty()) { + auto n = std::min(chunkDist(rng), remaining.size()); + chunkedSink(remaining.substr(0, n)); + remaining = remaining.substr(n); + } + } + chunkedSink.flush(); + + ASSERT_EQ(singleOut.s, expectedOutput); + ASSERT_EQ(chunkedOut.s, expectedOutput); + ASSERT_EQ(singleSink.matches, expectedMatches); + ASSERT_EQ(chunkedSink.matches, expectedMatches); +} + TEST(references, scan) { std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0"; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 51bcb29aa903..5126f243a5b2 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -22,7 +22,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) auto tmpRoot = createTempDir(); auto realStoreDir = tmpRoot / "nix/store"; - std::filesystem::create_directories(realStoreDir); + createDirs(realStoreDir); std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); auto localStore = std::dynamic_pointer_cast(store); @@ -34,12 +34,13 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) std::string drvName = fmt("register-valid-paths-bench-%d", i); auto drvPath = StorePath::random(drvName + ".drv"); - Derivation drv; - drv.name = drvName; - drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}}); - drv.platform = "x86_64-linux"; - drv.builder = "foo"; - drv.env["out"] = ""; + Derivation drv{ + .outputs = {{"out", DerivationOutput{DerivationOutput::Deferred{}}}}, + .platform = "x86_64-linux", + .builder = "foo", + .env = {{"out", ""}}, + .name = drvName, + }; drv.fillInOutputPaths(*localStore); auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false); @@ -66,7 +67,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) state.PauseTiming(); localStore.reset(); store.reset(); - std::filesystem::remove_all(tmpRoot); + deletePath(tmpRoot); state.ResumeTiming(); } diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 3534d44d0d8f..29a7e3a8320d 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -179,16 +179,17 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) EnableExperimentalFeature enableCA{"ca-derivations"}; // Create a CA floating output derivation - Derivation drv; - drv.name = "test-ca-drv"; - drv.outputs = { - { - "out", - DerivationOutput{DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }}, + Derivation drv{ + .outputs{ + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, }, + .name = "test-ca-drv", }; // Write the derivation to the destination store @@ -317,19 +318,20 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) }); // Create the root CA floating derivation that depends on depDrv - Derivation rootDrv; - rootDrv.name = "root-drv"; - rootDrv.outputs = { - { - "out", - DerivationOutput{DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }}, + Derivation rootDrv{ + .outputs{ + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, }, + // Add the dependency derivation as an input + .inputs = {.drvs = {.map = {{depDrvPath, {.value = {"out"}}}}}}, + .name = "root-drv", }; - // Add the dependency derivation as an input - rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; // Write the root derivation to the destination store auto rootDrvPath = dummyStore->writeDerivation(rootDrv); @@ -345,7 +347,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) ASSERT_TRUE(resolvedRootDrv); // Write the resolved derivation to the substituter - auto resolvedRootDrvPath = substituter->writeDerivation(Derivation{*resolvedRootDrv}); + auto resolvedRootDrvPath = substituter->writeDerivation(resolvedRootDrv->unresolve()); // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index 43060cb697b8..65b475387b54 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -31,15 +31,13 @@ class WriteDerivationTest : public LibStoreTest TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { - auto drv = []() { - Derivation drv; - drv.name = "simple-derivation"; - drv.platform = "system"; - drv.builder = "foo"; - drv.args = {"bar", "baz"}; - drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; - return drv; - }(); + Derivation drv{ + .platform = "system", + .builder = "foo", + .args = {"bar", "baz"}, + .env = {{"BIG_BAD", "WOLF"}}, + .name = "simple-derivation", + }; auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index f755717eb531..e4a909a4a554 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -28,12 +28,16 @@ namespace nix { +void AwsAuthError::anchor() {} + AwsAuthError::AwsAuthError(int errorCode) : CloneableError("AWS authentication error: '%s' (%d)", aws_error_str(errorCode), errorCode) , errorCode(errorCode) { } +AwsCredentialProvider::~AwsCredentialProvider() {} + namespace { /** @@ -284,8 +288,6 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr makeAwsCredentialsProvider() { return make_ref(); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 64fe33536bbd..5294ee7a0330 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -12,16 +12,24 @@ #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" +#include "nix/util/util.hh" #include #include #include #include +#include #include namespace nix { +void BinaryCacheStoreConfig::anchor() {} + +void BinaryCacheStore::anchor() {} + +void NoSuchBinaryCacheFile::anchor() {} + BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { @@ -134,8 +142,7 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) std::shared_ptr(narInfo)); } -ref BinaryCacheStore::addToStoreCommon( - Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo) +ref BinaryCacheStore::uploadData(Source & narSource, RepairFlag repair, fun mkInfo) { auto fdTemp = createAnonymousTempFile(); @@ -165,7 +172,7 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = config.compression.to_string(); // FIXME: Make NarInfo use CompressionAlgo + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; @@ -187,19 +194,6 @@ ref BinaryCacheStore::addToStoreCommon( ((1.0 - (double) fileSize / info.narSize) * 100.0), duration); - /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) - try { - if (ref != info.path) - queryPathInfo(ref); - } catch (InvalidPath &) { - throw Error( - "cannot add '%s' to the binary cache because the reference '%s' is not valid", - printStorePath(info.path), - printStorePath(ref)); - } - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (config.writeNARListing) { @@ -280,24 +274,45 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; + return narInfo; +} + +void BinaryCacheStore::uploadNarInfo(ref narInfo) +{ + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : narInfo->references) + try { + if (ref != narInfo->path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error( + "cannot add '%s' to the binary cache because the reference '%s' is not valid", + printStorePath(narInfo->path), + printStorePath(ref)); + } + narInfo->sign(*this, signers); /* Atomically write the NAR info file.*/ writeNarInfo(narInfo); stats.narInfoWrite++; +} +ref BinaryCacheStore::addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo) +{ + auto narInfo = uploadData(narSource, repair, std::move(mkInfo)); + uploadNarInfo(narInfo); return narInfo; } void BinaryCacheStore::addToStore( const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) { - if (!repair && isValidPath(info.path)) { - // FIXME: copyNAR -> null sink - narSource.drain(); + if (!repair && isValidPath(info.path)) return; - } addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { /* FIXME reinstate these, once we can correctly do hash modulo sink as @@ -308,6 +323,137 @@ void BinaryCacheStore::addToStore( }}); } +void BinaryCacheStore::addMultipleToStore( + PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) +{ + /* Index the paths to copy by store path so the graph nodes below + can look up each path's info (NAR size, references) and source. */ + std::map> *> infosMap; + uint64_t bytesExpected = 0; + for (auto & item : pathsToCopy) { + bytesExpected += item.first.narSize; + infosMap.insert_or_assign(item.first.path, &item); + } + act.setExpected(actCopyPath, bytesExpected); + + std::atomic nrDone{0}; + std::atomic nrRunning{0}; + auto showProgress = [&, nrTotal = pathsToCopy.size()]() { act.progress(nrDone, nrTotal, nrRunning); }; + + /* The NarInfos produced by uploading the NARs, to be consumed when + writing the .narinfo files. Populated by the `UploadNar` nodes + and read by the corresponding `UploadNarInfo` nodes. */ + Sync>> narInfos_; + + /* The work graph has two kinds of nodes: uploading the NAR for a + path (which has no dependencies, since NARs are independent of + each other), and uploading the .narinfo for a path (which depends + on the corresponding NAR upload and on the .narinfo uploads of all + the path's references). Processing the latter in topological order + maintains the closure invariant: whenever a .narinfo exists, the + .narinfo files of all its references exist as well. */ + struct UploadNar + { + StorePath path; + uint64_t narSize; + + /* Order NAR uploads by descending size so that the largest + (and typically slowest) NARs are started first. */ + bool operator<(const UploadNar & other) const + { + return narSize != other.narSize ? narSize > other.narSize : path < other.path; + } + }; + + struct UploadNarInfo + { + StorePath path; + + bool operator<(const UploadNarInfo & other) const + { + return path < other.path; + } + }; + + /* `std::variant`'s `operator<` orders by alternative index first, so + all `UploadNar` nodes sort (and thus get enqueued) before any + `UploadNarInfo` node. + TODO: uploading the debug info and NAR listings could be turned into separate graph nodes as well. + */ + using Node = std::variant; + + std::set nodes; + for (auto & [path, item] : infosMap) { + nodes.insert(UploadNar{path, item->first.narSize}); + nodes.insert(UploadNarInfo{path}); + } + + processGraph( + nodes, + + [&](const Node & node) -> std::set { + return std::visit( + overloaded{ + [&](const UploadNar &) -> std::set { + /* NAR uploads have no dependencies. */ + return {}; + }, + [&](const UploadNarInfo & n) -> std::set { + std::set edges; + auto & info = infosMap.at(n.path)->first; + /* Wait for our own NAR to be uploaded ... */ + edges.insert(UploadNar{n.path, info.narSize}); + /* ... and for the .narinfo files of all + references that are part of this copy (other + references are already valid in the store). */ + for (auto & ref : info.references) { + if (ref != n.path && infosMap.count(ref)) + edges.insert(UploadNarInfo{ref}); + } + return edges; + }, + }, + node); + }, + + [&](const Node & node) { + checkInterrupt(); + std::visit( + overloaded{ + [&](const UploadNar & n) { + auto & [info, source_] = *infosMap.at(n.path); + + /* Make sure the Source object is destroyed when + we're done, e.g. to release the connection + lock held by LegacySSHStore::narFromPath(). */ + auto source = std::move(source_); + + if (repair || !isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + auto narInfo = uploadData(*source, repair, [&](HashResult nar) { + auto info2 = info; + info2.ultimate = false; + return info2; + }); + narInfos_.lock()->insert_or_assign(info.path, narInfo); + } + + nrDone++; + showProgress(); + }, + [&](const UploadNarInfo & n) { + auto & info = infosMap.at(n.path)->first; + if (!repair && isValidPath(info.path)) + return; + auto narInfo = narInfos_.lock()->at(n.path); + uploadNarInfo(narInfo); + }, + }, + node); + }); +} + StorePath BinaryCacheStore::addToStoreFromDump( Source & dump, std::string_view name, @@ -419,7 +565,13 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize; }}; - auto decompressor = makeDecompressionSink(info->compression, uncompressedSink); + /* makeDecompressionSink used to treat empty strings as "none". It seems + impossible that it would actually end up here with an empty string though + (since an empty `Compression: ' is treated as bzip2 when parsed from a + .narinfo file and the narinfo disk cache wouldn't handle empty strings). + TODO: Revisit this and convert to an assert probably or even made + compression a non-optional field. */ + auto decompressor = makeDecompressionSink(info->compression.value_or(CompressionAlgo::none), uncompressedSink); try { getFile(info->url, *decompressor); diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index f01911bcd002..7dd6954f607c 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -4,6 +4,8 @@ namespace nix { +void BuildError::anchor() {} + void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) { // Allow selecting a subset of enum values @@ -20,7 +22,7 @@ void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) checkMismatch = true; break; case BuildResult::Failure::PermanentFailure: - // Also considered a permenant failure, it seems + // Also considered a permanent failure, it seems case BuildResult::Failure::InputRejected: permanentFailure = true; break; @@ -143,10 +145,10 @@ std::strong_ordering BuildError::operator<=>(const BuildError & other) const noe namespace nlohmann { -using namespace nix; - -void adl_serializer::to_json(json & res, const BuildResult & br) +void adl_serializer::to_json(json & res, const nix::BuildResult & br) { + using namespace nix; + res = json::object(); // Common fields @@ -179,8 +181,10 @@ void adl_serializer::to_json(json & res, const BuildResult & br) br.inner); } -BuildResult adl_serializer::from_json(const json & _json) +nix::BuildResult adl_serializer::from_json(const json & _json) { + using namespace nix; + auto & json = getObject(_json); BuildResult br; @@ -217,8 +221,10 @@ BuildResult adl_serializer::from_json(const json & _json) return br; } -KeyedBuildResult adl_serializer::from_json(const json & json0) +nix::KeyedBuildResult adl_serializer::from_json(const json & json0) { + using namespace nix; + auto json = getObject(json0); return KeyedBuildResult{ @@ -227,8 +233,9 @@ KeyedBuildResult adl_serializer::from_json(const json & json0) }; } -void adl_serializer::to_json(json & json, const KeyedBuildResult & kbr) +void adl_serializer::to_json(json & json, const nix::KeyedBuildResult & kbr) { + using namespace nix; adl_serializer::to_json(json, kbr); json["path"] = kbr.path; } diff --git a/src/libstore/build/build-log.cc b/src/libstore/build/build-log.cc index a8fb64fc68bf..920affe12afe 100644 --- a/src/libstore/build/build-log.cc +++ b/src/libstore/build/build-log.cc @@ -10,9 +10,16 @@ BuildLog::BuildLog(size_t maxTailLines, std::unique_ptr act) void BuildLog::operator()(std::string_view data) { - for (auto c : data) + for (auto c : data) { + /* Only let a '\r' reset the column if it isn't followed by '\n', so + "\r\n" acts as a line terminator; defer a char to handle split chunks. */ + if (pendingCR) { + pendingCR = false; + if (c != '\n') + currentLogLinePos = 0; + } if (c == '\r') - currentLogLinePos = 0; + pendingCR = true; else if (c == '\n') flushLine(); else { @@ -20,6 +27,7 @@ void BuildLog::operator()(std::string_view data) currentLogLine.resize(currentLogLinePos + 1); currentLogLine[currentLogLinePos++] = c; } + } } void BuildLog::flush() diff --git a/src/libstore/build/derivation-builder.cc b/src/libstore/build/derivation-builder.cc index 39ac40175f71..a38f7b2bc029 100644 --- a/src/libstore/build/derivation-builder.cc +++ b/src/libstore/build/derivation-builder.cc @@ -3,10 +3,9 @@ namespace nlohmann { -using namespace nix; - -ExternalBuilder adl_serializer::from_json(const json & json) +nix::ExternalBuilder adl_serializer::from_json(const json & json) { + using namespace nix; auto obj = getObject(json); return { .systems = valueAt(obj, "systems"), @@ -15,7 +14,7 @@ ExternalBuilder adl_serializer::from_json(const json & json) }; } -void adl_serializer::to_json(json & json, const ExternalBuilder & eb) +void adl_serializer::to_json(json & json, const nix::ExternalBuilder & eb) { json = { {"systems", eb.systems}, @@ -25,3 +24,13 @@ void adl_serializer::to_json(json & json, const ExternalBuilder } } // namespace nlohmann + +namespace nix { + +void BuilderFailureError::anchor() {} + +void DerivationBuilder::anchor() {} + +DerivationBuilderCallbacks::~DerivationBuilderCallbacks() {} + +} // namespace nix diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 57a4017e8e24..e239dff8e203 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -1,5 +1,7 @@ #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-env-desugar.hh" +#include "nix/store/restricted-store.hh" +#include "nix/store/daemon.hh" #ifndef _WIN32 // TODO enable build hook on Windows # include "nix/store/build/hook-instance.hh" # include "nix/store/build/derivation-builder.hh" @@ -30,8 +32,8 @@ namespace nix { DerivationBuildingGoal::DerivationBuildingGoal( - const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode, bool storeDerivation) - : Goal(worker, gaveUpOnSubstitution(storeDerivation)) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) + : Goal(worker, gaveUpOnSubstitution()) , drvPath(drvPath) , drv{std::move(drv)} , buildMode(buildMode) @@ -51,7 +53,8 @@ std::string DerivationBuildingGoal::key() return "dd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv) +template +std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv) { std::string msg; StorePathSet expectedOutputPaths; @@ -66,6 +69,11 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } +template std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv); +template std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv); + +namespace { + struct LogSink : Sink { Activity & act; @@ -102,6 +110,8 @@ struct LogSink : Sink } }; +} // namespace + struct PostBuildHookState { const std::string hook; @@ -140,7 +150,7 @@ static std::unique_ptr runPostBuildHook( /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) +Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() { Goals waitees; @@ -151,13 +161,13 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) are (resolved) derivation outputs in a resolved derivation. */ if (&worker.evalStore != &worker.store) { RealisedPath::Set inputSrcs; - for (auto & i : drv->inputSrcs) + for (auto & i : drv->inputs) if (worker.evalStore.isValidPath(i)) inputSrcs.insert(i); copyClosure(worker.evalStore, worker.store, inputSrcs); } - for (auto & i : drv->inputSrcs) { + for (auto & i : drv->inputs) { if (worker.store.isValidPath(i)) continue; if (!worker.settings.useSubstitutes) @@ -188,47 +198,8 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) /* Determine the full set of input paths. */ - if (storeDerivation) { - assert(drv->inputDrvs.map.empty()); - /* Store the resolved derivation, as part of the record of - what we're actually building */ - worker.store.writeDerivation(*drv); - } - StorePathSet inputPaths; - - { - /* If we get this far, we know no dynamic drvs inputs */ - - for (auto & [depDrvPath, depNode] : drv->inputDrvs.map) { - for (auto & outputName : depNode.value) { - /* Don't need to worry about `inputGoals`, because - impure derivations are always resolved above. Can - just use DB. This case only happens in the (older) - input addressed and fixed output derivation cases. */ - auto outMap = [&] { - for (auto * drvStore : {&worker.evalStore, &worker.store}) - if (drvStore->isValidPath(depDrvPath)) - return deepQueryDerivationOutputMap(worker.store, depDrvPath, drvStore); - assert(false); - }(); - - auto outMapPath = outMap.find(outputName); - if (outMapPath == outMap.end()) { - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), - outputName, - worker.store.printStorePath(depDrvPath)); - } - - worker.store.computeFSClosure(outMapPath->second, inputPaths); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + worker.store.computeFSClosure(drv->inputs, inputPaths); debug("added input paths %s", concatMapStringsSep(", ", inputPaths, [&](auto & p) { return "'" + worker.store.printStorePath(p) + "'"; @@ -333,35 +304,12 @@ static BuildError reject(const LocalBuildRejection & rejection, std::string_view Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) { auto drvOptions = [&] { - DerivationOptions temp; try { - temp = - derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); + return derivationOptionsFromStructuredAttrs(worker.store, drv->env, get(drv->structuredAttrs)); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); throw; } - - auto res = tryResolve( - temp, - [&](ref drvPath, const std::string & outputName) -> std::optional { - try { - return resolveDerivedPath( - worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore); - } catch (Error &) { - return std::nullopt; - } - }); - - /* The derivation must have all of its inputs gotten this point, - so the resolution will surely succeed. - - (Actually, we shouldn't even enter this goal until we have a - resolved derivation, or derivation with only input addressed - transitive inputs, so this should be a no-opt anyways.) - */ - assert(res); - return *res; }(); std::map initialOutputs; @@ -651,12 +599,14 @@ Goal::Co DerivationBuildingGoal::buildWithHook( destroyed (e.g., during failure cascades). */ hook->onKillChild = [this]() { worker.childTerminated(this, JobCategory::Build); }; - try { - hook->machineName = readLine(hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the machine name from the build hook"); - throw; - } + std::string machineName = [&hook]() { + try { + return readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw; + } + }(); CommonProto::WriteConn conn{hook->sink}; @@ -695,16 +645,12 @@ Goal::Co DerivationBuildingGoal::buildWithHook( : buildMode == bmCheck ? "checking outputs of '%s'" : "building '%s'", worker.store.printStorePath(drvPath)); - msg += fmt(" on '%s'", hook->machineName); + msg += fmt(" on '%s'", machineName); std::unique_ptr buildLog = std::make_unique( worker.settings.logLines, std::make_unique( - *logger, - lvlInfo, - actBuild, - msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1})); + *logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), machineName, 1, 1})); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); @@ -764,9 +710,9 @@ Goal::Co DerivationBuildingGoal::buildWithHook( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -938,6 +884,26 @@ Goal::Co DerivationBuildingGoal::buildLocally( { closeLogFileFn(); } + + void processDaemonConnection( + ref store, + FdSource && from, + FdSink && to, + RestrictionContext & context, + daemon::RecursiveFlag recursiveFlag) override + { + /** + * TODO: We create a fresh Worker here because the + * parent Worker is blocked waiting for the current + * build to finish, so we can't reuse it from a + * daemon thread. Ideally we should reuse the same + * Worker to share scheduling state. + */ + Worker freshWorker{goal.worker.store, goal.worker.evalStore}; + auto builder = makeRestrictedBuilder(freshWorker, context); + daemon::processConnection( + store, std::move(from), std::move(to), NotTrusted, recursiveFlag, builder.get_ptr()); + } }; decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = @@ -986,12 +952,12 @@ Goal::Co DerivationBuildingGoal::buildLocally( builder = localBuildCap.externalBuilder ? makeExternalDerivationBuilder( localBuildCap.localStore, - std::make_unique(*this, openLogFile, closeLogFile), + std::make_shared(*this, openLogFile, closeLogFile), std::move(params), *localBuildCap.externalBuilder) : makeDerivationBuilder( localBuildCap.localStore, - std::make_unique(*this, openLogFile, closeLogFile), + std::make_shared(*this, openLogFile, closeLogFile), std::move(params)); } @@ -1035,9 +1001,9 @@ Goal::Co DerivationBuildingGoal::buildLocally( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -1121,7 +1087,6 @@ static std::unique_ptr runPostBuildHook( hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); ProcessOptions processOptions; - processOptions.allowVfork = false; state->pid = startProcess( [&] { diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index dffbbc3beb26..d1562e811c7e 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -1,16 +1,88 @@ #include +#include "nix/store/derivations.hh" #include "nix/store/store-api.hh" #include "nix/store/build-result.hh" +#include "nix/util/hash.hh" #include "derivation-check.hh" namespace nix { +void checkCAOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + const std::string & outputName) +{ + std::visit( + overloaded{ + [&](const DerivationOutput::CAFixed & dof) { + auto & wanted = dof.ca.hash; + + /* Check wanted hash */ + assert(info.ca); + auto & got = info.ca->hash; + if (wanted != got) { + throw BuildError( + BuildResult::Failure::HashMismatch, + "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true)); + } + if (!info.references.empty()) { + auto numViolations = info.references.size(); + throw BuildError( + BuildResult::Failure::HashMismatch, + "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*info.references.begin())); + } + }, + [&](const DerivationOutput::CAFloating & dof) { + if (!info.ca.has_value()) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "floating content-addressing derivation '%s' output '%s' (at '%s') was not content-addressed", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path)); + } + if (info.ca->method != dof.method) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "content-addressing derivation '%s' output '%s' (at '%s') was hashed with method '%s', expected '%s'", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path), + info.ca->method.render(), + dof.method.render()); + } + if (info.ca->hash.algo != dof.hashAlgo) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "content-addressing derivation '%s' output '%s' (at '%s') was hashed with algorithm '%s', expected '%s'", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path), + printHashAlgo(info.ca->hash.algo), + printHashAlgo(dof.hashAlgo)); + } + }, + [&](const DerivationOutput::Deferred & _) {}, + [&](const DerivationOutput::Impure & _) {}, + [&](const DerivationOutput::InputAddressed & _) {}, + }, + outputSpec.raw); +} + void checkOutputs( Store & store, const StorePath & drvPath, - const decltype(Derivation::outputs) & drvOutputs, + const BasicDerivation & drv, const decltype(DerivationOptions::outputChecks) & outputChecks, const std::map & outputs) { @@ -24,36 +96,29 @@ void checkOutputs( const std::string & outputName = pair.first; const auto & info = pair.second; - auto * outputSpec = get(drvOutputs, outputName); - assert(outputSpec); - - if (const auto * dof = std::get_if(&outputSpec->raw)) { - auto & wanted = dof->ca.hash; + auto * outputSpec = get(drv.outputs, outputName); + if (!outputSpec) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "builder for '%s' submitted unknown output '%s' (Valid outputs are [%s])", + store.printStorePath(drvPath), + outputName, + concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; })); + } - /* Check wanted hash */ - assert(info.ca); - auto & got = info.ca->hash; - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - throw BuildError( - BuildResult::Failure::HashMismatch, - "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true)); - } - if (!info.references.empty()) { - auto numViolations = info.references.size(); - throw BuildError( - BuildResult::Failure::HashMismatch, - "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*info.references.begin())); - } + if (outputPathName(drv.name, outputName) != info.path.name()) { + throw BuildError( + BuildResult::Failure::OutputRejected, + "derivation '%s' output '%s' (at '%s') was named '%s', expected '%s'", + store.printStorePath(drvPath), + outputName, + store.printStorePath(info.path), + info.path.name(), + outputPathName(drv.name, outputName)); } + checkCAOutput(store, drvPath, *outputSpec, info, outputName); + /* Compute the closure and closure size of some output. This is slightly tricky because some of its references (namely other outputs) may not be valid yet. */ @@ -118,8 +183,6 @@ void checkOutputs( if (auto output = get(outputs, refOutputName)) spec.insert(output->path); else { - std::string outputsListing = - concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); throw BuildError( BuildResult::Failure::OutputRejected, "derivation '%s' output check for '%s' contains output name '%s'," @@ -128,7 +191,7 @@ void checkOutputs( store.printStorePath(drvPath), outputName, refOutputName, - outputsListing); + concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; })); } }}, i); diff --git a/src/libstore/build/derivation-check.hh b/src/libstore/build/derivation-check.hh index 01e6c5d56383..5c8c75da172d 100644 --- a/src/libstore/build/derivation-check.hh +++ b/src/libstore/build/derivation-check.hh @@ -7,6 +7,18 @@ namespace nix { +/** + * If outputSpec is a CAFixed or CAFloating output, check that the actual output described in + * info meets the requirements for a CA output. + * Do nothing if outputSpec is not a CAFixed or CAFloating output. + */ +void checkCAOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + const std::string & outputName); + /** * Check that outputs meets the requirements specified by the * 'outputChecks' attribute (or the legacy @@ -20,7 +32,7 @@ namespace nix { void checkOutputs( Store & store, const StorePath & drvPath, - const decltype(Derivation::outputs) & drvOutputs, + const BasicDerivation & drv, const decltype(DerivationOptions::outputChecks) & drvOptions, const std::map & outputs); diff --git a/src/libstore/build/derivation-env-desugar.cc b/src/libstore/build/derivation-env-desugar.cc index 75b62c116502..ff19472d0122 100644 --- a/src/libstore/build/derivation-env-desugar.cc +++ b/src/libstore/build/derivation-env-desugar.cc @@ -19,7 +19,7 @@ std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fil DesugaredEnv DesugaredEnv::create( Store & store, - const Derivation & drv, + const BasicDerivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths) { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6e2d3223b10f..19281191cc7f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,7 +52,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto drvOptions = [&]() -> DerivationOptions { try { return derivationOptionsFromStructuredAttrs( - worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); + worker.store, drv->inputs.drvs, drv->env, get(drv->structuredAttrs)); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); throw; @@ -92,6 +92,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) co_await await(std::move(waitees)); if (nrFailed == 0) { + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); co_await await(std::move(waitees)); @@ -111,6 +113,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } } + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); @@ -144,13 +148,15 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) worker.store.printStorePath(drvPath)); } - auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode); - { - Goals waitees{resolutionGoal}; - co_await await(std::move(waitees)); - } + auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, drv, buildMode); + /* We'll handle the error below. */ + resolutionGoal->preserveFailure = true; + co_await await({resolutionGoal}); + if (nrFailed != 0) { - co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); + auto * failure = resolutionGoal->buildResult.tryGetFailure(); + assert(failure); + co_return doneFailure(*failure); } if (resolutionGoal->resolvedDrv) { @@ -158,7 +164,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, - make_ref(drvResolved), + make_ref(drvResolved.unresolve()), wantedOutput, buildMode, /*storeDerivation=*/true); @@ -219,18 +225,64 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(false); } + /* We don't need it any more and don't want to hold on to it while suspended. */ + resolutionGoal.reset(); + /* Give up on substitution for the output we want, actually build this derivation */ - auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); + /* Project down to the `BasicDerivation` the builder consumes, + adding the outputs of the input derivations to the input + sources. */ + auto resolvedDrv = make_ref(drv->mapInputs([&](const FullInputs & inputs) { + auto srcs = inputs.srcs; + for (auto & [depDrvPath, depNode] : inputs.drvs.map) { + for (auto & outputName : depNode.value) { + /* Don't need to worry about `inputGoals`, because + impure derivations are always resolved above. Can + just use DB. This case only happens in the (older) + input addressed and fixed output derivation cases. */ + auto outMap = [&] { + for (auto * drvStore : {&worker.evalStore, &worker.store}) + if (drvStore->isValidPath(depDrvPath)) + return deepQueryDerivationOutputMap(worker.store, depDrvPath, drvStore); + assert(false); + }(); + auto outMapPath = outMap.find(outputName); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), + outputName, + worker.store.printStorePath(depDrvPath)); + } + srcs.insert(outMapPath->second); + } + } + return srcs; + })); + + if (storeDerivation) { + assert(drv->inputs.drvs.map.empty()); + /* `writeDerivation` checks the derivation's references are valid, + so the eval store's sources must be copied over first. */ + if (&worker.evalStore != &worker.store) { + RealisedPath::Set inputSrcs; + for (auto & i : resolvedDrv->inputs) + if (worker.evalStore.isValidPath(i)) + inputSrcs.insert(i); + copyClosure(worker.evalStore, worker.store, inputSrcs); + } + /* Store the resolved derivation, as part of the record of + what we're actually building */ + worker.store.writeDerivation(resolvedDrv->unresolve()); + } + + auto g = worker.makeDerivationBuildingGoal(drvPath, resolvedDrv, buildMode); /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; - { - Goals waitees; - waitees.insert(g); - co_await await(std::move(waitees)); - } + co_await await({g}); trace("outer build done"); diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 81c698e18563..eac55a2e8f3f 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -7,10 +7,10 @@ namespace nix { DerivationResolutionGoal::DerivationResolutionGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) : Goal(worker, resolveDerivation()) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv(std::move(drv)) , buildMode{buildMode} { name = fmt("resolving derivation '%s'", worker.store.printStorePath(drvPath)); @@ -22,60 +22,49 @@ std::string DerivationResolutionGoal::key() return "dc$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -/** - * Used for `inputGoals` local variable below - */ -struct value_comparison -{ - template - bool operator()(const ref & lhs, const ref & rhs) const - { - return *lhs < *rhs; - } -}; - Goal::Co DerivationResolutionGoal::resolveDerivation() { Goals waitees; - std::map, GoalPtr, value_comparison> inputGoals; - - { - std::function, const DerivedPathMap::ChildNode &)> - addWaiteeDerivedPath; - - addWaiteeDerivedPath = [&](ref inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto g = worker.makeGoal( - DerivedPath::Built{ - .drvPath = inputDrv, - .outputs = inputNode.value, - }, - buildMode == bmRepair ? bmRepair : bmNormal); - inputGoals.insert_or_assign(inputDrv, g); - waitees.insert(std::move(g)); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - addWaiteeDerivedPath( - make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); - }; - - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { - /* Ensure that pure, non-fixed-output derivations don't - depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() - && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (inputDrv.type().isImpure()) - throw Error( - "pure derivation '%s' depends on impure derivation '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(inputDrvPath)); - } - - addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); + using ValueComparison = decltype([](const ref & lhs, const ref & rhs) { + /* Compare the values, not the pointers themselves. */ + return *lhs < *rhs; + }); + + std::map, GoalPtr, ValueComparison> inputGoals; + + auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( + this const auto & self, + ref inputDrv, + const DerivedPathMap::ChildNode & inputNode) -> void { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( + DerivedPath::Built{ + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); } + for (const auto & [outputName, childNode] : inputNode.childMap) + self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : drv->inputs.drvs.map) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() + && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) + throw Error( + "pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(inputDrvPath)); + } + + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } co_await await(std::move(waitees)); @@ -144,7 +133,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() } assert(attempt); - auto pathResolved = computeStorePath(worker.store, Derivation{*attempt}); + auto pathResolved = computeStorePath(worker.store, attempt->unresolve()); auto msg = fmt("resolved derivation: '%s' -> '%s'", diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index edf8d1e86ebc..14864ed052f8 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -2,6 +2,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/derivations.hh" +#include +#include + namespace nix { DerivationTrampolineGoal::DerivationTrampolineGoal( @@ -144,11 +147,14 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation }, wantedOutputs.raw); + /* Must have at least one wanted output. This is assumed below. */ + assert(!resolvedWantedOutputs.empty()); + Goals concreteDrvGoals; /* Build this step! */ - auto sharedDrv = make_ref(std::move(drv)); + auto sharedDrv = make_ref(std::move(drv)); for (auto & output : resolvedWantedOutputs) { auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); @@ -157,20 +163,71 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation concreteDrvGoals.insert(std::move(g)); } - // Copy on purpose - co_await await(Goals(concreteDrvGoals)); + co_await await(concreteDrvGoals); trace("outer build done"); - auto & g = *concreteDrvGoals.begin(); - buildResult = g->buildResult; - if (auto * successP = buildResult.tryGetSuccess()) - for (auto & g2 : concreteDrvGoals) - if (auto * successP2 = g2->buildResult.tryGetSuccess()) - for (auto && [x, y] : successP2->builtOutputs) - successP->builtOutputs.insert_or_assign(x, y); + if (nrFailed != 0) { + auto gi = std::ranges::find_if(concreteDrvGoals, [](const GoalPtr & goal) -> bool { + auto exitCode = goal->exitCode; + /* Note that without --keep-going waitees might be cancelled before + we are woken up. */ + return exitCode != ecBusy && exitCode != ecSuccess; + }); + + const Goal * g = gi->get(); + assert(gi != concreteDrvGoals.end() && "expected a failing goal"); + auto exitCode = g->exitCode; + const auto * failure = g->buildResult.tryGetFailure(); + assert(failure && "failing goal does not report a failed build result"); + + /* Report the exit status of *some* failing goal. This might not be strictly + correct, since multiple subgoals can fail independently, but this should be + a good enough heuristic without --keep-going. */ + co_return doneFailure(exitCode, *failure); + } + + SingleDrvOutputs outputs; + + auto successes = std::views::transform(concreteDrvGoals, [](const GoalPtr & a) -> const BuildResult::Success & { + auto * success = a->buildResult.tryGetSuccess(); + assert(success && "goal succeeded, but some waitees do not report a successful status"); + return *success; + }); + + for (const auto & success : successes) + std::ranges::copy(success.builtOutputs, std::inserter(outputs, outputs.end())); + + auto statuses = successes | std::views::transform(&BuildResult::Success::status); + + /* Aggregate the status code. If some outputs we already valid, but we had + to build/substitute the other ones, report it as the smallest common + denominator. */ + auto compareSuccesses = [](auto a, auto b) { + /* This is technically an identity mapping of the underlying values, but + it would be worse to rely on the enum ordering here. */ + auto toPriority = [](auto st) { + using enum BuildResult::Success::Status; + switch (st) { + case Built: + return 0; + case Substituted: + return 1; + case AlreadyValid: + return 2; + case ResolvesToAlreadyValid: + return 3; + default: + unreachable(); + } + }; + return toPriority(a) < toPriority(b); + }; - co_return amDone(g->exitCode); + co_return doneSuccess({ + .status = std::ranges::min(statuses, compareSuccesses), + .builtOutputs = std::move(outputs), + }); } } // namespace nix diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 5b97966847b8..bf919e443e6b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,20 +1,63 @@ #include "nix/store/derivations.hh" #include "nix/store/build/worker.hh" +#include "nix/store/worker-settings.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" +#include "nix/store/store-open.hh" #include "nix/util/strings.hh" +#include namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) +void LocalBuilder::buildPaths(const std::vector & reqs, BuildMode buildMode) { - Worker worker(*this, evalStore ? *evalStore : *this); + getWorker()->buildPaths(reqs, buildMode); +} + +std::vector +LocalBuilder::buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) +{ + return getWorker()->buildPathsWithResults(reqs, buildMode); +} + +BuildResult LocalBuilder::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +{ + return getWorker()->buildDerivation(drvPath, drv, buildMode); +} + +BuildResult LocalBuilder::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, const StorePathSet & inputs, BuildMode buildMode) +{ + return getWorker()->buildDerivation(drvPath, drv, inputs, buildMode); +} + +std::vector LocalBuilder::buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) +{ + return getWorker()->buildPathsWithResults(reqs, inputs, buildMode); +} + +void LocalBuilder::ensurePath(const StorePath & path) +{ + /* If the path is already valid, we're done. */ + if (store->isValidPath(path)) + return; + + getWorker()->ensurePath(path); +} +void LocalBuilder::repairPath(const StorePath & path) +{ + getWorker()->repairPath(path); +} + +void Worker::buildPaths(const std::vector & reqs, BuildMode buildMode) +{ Goals goals; for (auto & br : reqs) - goals.insert(worker.makeGoal(br, buildMode)); + goals.insert(makeGoal(br, buildMode)); - worker.run(goals); + run(goals); StringSet failed; BuildResult::Failure * failure = nullptr; @@ -27,38 +70,35 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvReq->to_string(*this)); + failed.insert(i2->drvReq->to_string(store)); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(printStorePath(i2->storePath)); + failed.insert(store.printStorePath(i2->storePath)); } } if (failed.size() == 1 && failure) { - failure->withExitStatus(worker.exitStatusFlags.failingExitStatus()); + failure->withExitStatus(exitStatusFlags.failingExitStatus()); throw *failure; } else if (!failed.empty()) { - auto exitStatus = worker.exitStatusFlags.failingExitStatus(); + auto exitStatus = exitStatusFlags.failingExitStatus(); if (failure) logError(failure->info()); throw Error(exitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } -std::vector Store::buildPathsWithResults( - const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) +std::vector Worker::buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) { - Worker worker(*this, evalStore ? *evalStore : *this); - Goals goals; std::vector> state; for (const auto & req : reqs) { - auto goal = worker.makeGoal(req, buildMode); + auto goal = makeGoal(req, buildMode); goals.insert(goal); state.push_back({req, goal}); } - worker.run(goals); + run(goals); std::vector results; results.reserve(state.size()); @@ -79,13 +119,12 @@ std::vector Store::buildPathsWithResults( return results; } -BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) +BuildResult Worker::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - Worker worker(*this, *this); - auto goal = worker.makeDerivationTrampolineGoal(drvPath, OutputsSpec::All{}, drv, buildMode); + auto goal = makeDerivationTrampolineGoal(drvPath, OutputsSpec::All{}, drv.unresolve(), buildMode); try { - worker.run(Goals{goal}); + run(Goals{goal}); return goal->buildResult; } catch (Error & e) { return BuildResult{ @@ -96,49 +135,65 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat }; } -void Store::ensurePath(const StorePath & path) +BuildResult Worker::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, const StorePathSet & inputs, BuildMode buildMode) +{ + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, store, inputs, NoRepair, NoCheckSigs, substitute); + return buildDerivation(drvPath, drv, buildMode); +} + +std::vector +Worker::buildPathsWithResults(const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) +{ + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + auto srcStore = openStore(); + copyPaths(*srcStore, store, inputs, NoRepair, NoCheckSigs, substitute); + return buildPathsWithResults(reqs, buildMode); +} + +void Worker::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ - if (isValidPath(path)) + if (store.isValidPath(path)) return; - Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path); + GoalPtr goal = makePathSubstitutionGoal(path); Goals goals = {goal}; - worker.run(goals); + run(goals); if (goal->exitCode != Goal::ecSuccess) { - auto exitStatus = worker.exitStatusFlags.failingExitStatus(); + auto exitStatus = exitStatusFlags.failingExitStatus(); goal->buildResult.tryThrowBuildError(exitStatus); - throw Error(exitStatus, "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(exitStatus, "path '%s' does not exist and cannot be created", store.printStorePath(path)); } } -void Store::repairPath(const StorePath & path) +void Worker::repairPath(const StorePath & path) { - Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); + GoalPtr goal = makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; - worker.run(goals); + run(goals); if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { + auto info = store.queryPathInfo(path); + if (info->deriver && store.isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeGoal( + goals.insert(makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. .outputs = OutputsSpec::All{}, }, bmRepair)); - worker.run(goals); + run(goals); } else - throw Error(worker.exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", store.printStorePath(path)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 946314afe140..5b1be405d246 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -4,24 +4,29 @@ namespace nix { +void WorkerSettings::anchor() {} + TimedOut::TimedOut(time_t maxDuration) : CloneableError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration) , maxDuration(maxDuration) { } +void TimedOut::anchor() {} + +void Goal::anchor() {} + using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; -using ChildEvents = decltype(promise_type::childEvents); -void ChildEvents::pushChildEvent(ChildOutput event) +void Goal::ChildEvents::pushChildEvent(ChildOutput event) { if (childTimeout) return; // Already timed out, ignore childOutputs.push(std::move(event)); } -void ChildEvents::pushChildEvent(ChildEOF event) +void Goal::ChildEvents::pushChildEvent(ChildEOF event) { if (childTimeout) return; // Already timed out, ignore @@ -29,20 +34,20 @@ void ChildEvents::pushChildEvent(ChildEOF event) childEOF = std::move(event); } -void ChildEvents::pushChildEvent(TimedOut event) +void Goal::ChildEvents::pushChildEvent(TimedOut event) { // Timeout is immediate - flush pending events childOutputs = {}; childEOF.reset(); - childTimeout = std::move(event); + childTimeout = std::make_unique(std::move(event)); } -bool ChildEvents::hasChildEvent() const +bool Goal::ChildEvents::hasChildEvent() const { return !childOutputs.empty() || childEOF || childTimeout; } -Goal::ChildEvent ChildEvents::popChildEvent() +Goal::ChildEvent Goal::ChildEvents::popChildEvent() { if (!childOutputs.empty()) { auto event = std::move(childOutputs.front()); @@ -52,23 +57,28 @@ Goal::ChildEvent ChildEvents::popChildEvent() if (childEOF) return *std::exchange(childEOF, std::nullopt); if (childTimeout) - return *std::exchange(childTimeout, std::nullopt); + return std::exchange(childTimeout, nullptr); unreachable(); } using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; -Co::Co(Co && rhs) +Co::Co(Co && rhs) noexcept { this->handle = rhs.handle; rhs.handle = nullptr; } -void Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) noexcept { - this->handle = rhs.handle; + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } + handle = rhs.handle; rhs.handle = nullptr; + return *this; } Co::~Co() @@ -273,22 +283,19 @@ void Goal::work() void Goal::handleChildOutput(Descriptor fd, std::string_view data) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); worker.wakeUp(shared_from_this()); } void Goal::handleEOF(Descriptor fd) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + childEvents.pushChildEvent(ChildEOF{fd}); worker.wakeUp(shared_from_this()); } void Goal::timedOut(TimedOut && ex) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + childEvents.pushChildEvent(std::move(ex)); worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 4cb42975fe29..90273493e288 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,7 +146,7 @@ Goal::Co PathSubstitutionGoal::init() } if (lastStoresException.has_value()) { if (!worker.settings.tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0c63beb4dd1e..2556a26f8593 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -61,7 +61,7 @@ std::shared_ptr Worker::initGoalIfNeeded(std::weak_ptr & goal_weak, Args & if (auto goal = goal_weak.lock()) return goal; - auto goal = std::make_shared(args...); + auto goal = std::make_shared(std::forward(args)...); goal_weak = goal; wakeUp(goal); return goal; @@ -104,16 +104,15 @@ std::shared_ptr Worker::makeDerivationGoal( } std::shared_ptr -Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode) +Worker::makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode); } -std::shared_ptr Worker::makeDerivationBuildingGoal( - const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation) +std::shared_ptr +Worker::makeDerivationBuildingGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { - return initGoalIfNeeded( - derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode, storeDerivation); + return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode); } std::shared_ptr @@ -295,28 +294,11 @@ void Worker::waitForCompletion(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; - - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back( - DerivedPath::Built{ - .drvPath = goal->drvReq, - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - store.queryMissing(topPaths); - debug("entered goal loop"); + for (std::shared_ptr goal : _topGoals) + topGoals.insert(std::move(goal)); while (1) { - checkInterrupt(); // TODO GC interface? diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 3dd66be2ccae..2ad98fa92741 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -10,6 +10,8 @@ namespace nix { +void BuildEnvFileConflictError::anchor() {} + RegisterBuiltinBuilder::BuiltinBuilders & RegisterBuiltinBuilder::builtinBuilders() { static RegisterBuiltinBuilder::BuiltinBuilders builders; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index d412adbb621f..fd38b5a4afd8 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -54,7 +54,8 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) } #endif - auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? CompressionAlgo::xz : CompressionAlgo::none, sink); fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index ee1d3bf8acde..db3677151416 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -9,6 +9,8 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authorit { } +void CommonSSHStoreConfig::anchor() {} + SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 497c2c5b47c1..fd8807d6d035 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,5 +1,6 @@ #include "nix/util/args.hh" #include "nix/store/content-address.hh" +#include "nix/util/file-content-address.hh" #include "nix/util/split.hh" #include "nix/util/json-utils.hh" @@ -133,6 +134,20 @@ FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const } } +FileSerialisationMethod ContentAddressMethod::getFileSerialisationMethod() const +{ + switch (raw) { + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::Text: + return FileSerialisationMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FileSerialisationMethod::NixArchive; + default: + assert(false); + } +} + std::string ContentAddress::render() const { return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); @@ -304,20 +319,19 @@ Hash ContentAddressWithReferences::getHash() const namespace nlohmann { -using namespace nix; - -ContentAddressMethod adl_serializer::from_json(const json & json) +nix::ContentAddressMethod adl_serializer::from_json(const json & json) { - return ContentAddressMethod::parse(getString(json)); + return nix::ContentAddressMethod::parse(nix::getString(json)); } -void adl_serializer::to_json(json & json, const ContentAddressMethod & m) +void adl_serializer::to_json(json & json, const nix::ContentAddressMethod & m) { json = m.render(); } -ContentAddress adl_serializer::from_json(const json & json) +nix::ContentAddress adl_serializer::from_json(const json & json) { + using namespace nix; auto obj = getObject(json); return { .method = adl_serializer::from_json(valueAt(obj, "method")), @@ -325,7 +339,7 @@ ContentAddress adl_serializer::from_json(const json & json) }; } -void adl_serializer::to_json(json & json, const ContentAddress & ca) +void adl_serializer::to_json(json & json, const nix::ContentAddress & ca) { json = { {"method", ca.method}, diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 52e1c121c3e7..f8c5815a29af 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,8 +1,11 @@ #include "nix/store/daemon.hh" +#include "nix/util/configuration.hh" +#include "nix/util/file-content-address.hh" #include "nix/util/signals.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" +#include "nix/store/build.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/filetransfer.hh" @@ -42,6 +45,8 @@ Sink & operator<<(Sink & sink, const Logger::Fields & fields) return sink; } +namespace { + /* Logger that forwards log messages to the client, *if* we're in a state where the protocol allows it (i.e., when canSendStderr is true). */ @@ -179,22 +184,6 @@ struct TunnelLogger : public Logger } }; -struct TunnelSink : Sink -{ - Sink & to; - - TunnelSink(Sink & to) - : to(to) - { - } - - void operator()(std::string_view data) override - { - to << STDERR_WRITE; - writeString(data, to); - } -}; - struct TunnelSource : BufferedSource { Source & from; @@ -217,6 +206,8 @@ struct TunnelSource : BufferedSource } }; +} // namespace + struct ClientSettings { bool keepFailed; @@ -318,11 +309,42 @@ static void performOp( TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::BasicServerConnection & conn, - WorkerProto::Op op) + WorkerProto::Op op, + Builder & builder) { WorkerProto::ReadConn rconn(conn); WorkerProto::WriteConn wconn(conn); + if (recursive == daemon::RecursiveFlag::RecursiveSubmitted) { + // Limit valid calls to reduce opportunities for nonreproducability in builds + // Since this is an allowlist, it's easiest to put it at the top before the switch + static constexpr std::array validOperations = { + // All the types of "Add" should be allowed + WorkerProto::Op::AddToStore, + WorkerProto::Op::AddMultipleToStore, + WorkerProto::Op::AddToStoreNar, + WorkerProto::Op::AddToStoreScanning, + // SubmitOutput is designed specifically for this use case + WorkerProto::Op::SubmitOutput, + // Used by nix cli, should never change actual outputs + WorkerProto::Op::AddTempRoot, + // Used by nix cli, restricted store will prevent it from seeing derivations it shouldn't + WorkerProto::Op::IsValidPath, + }; + if (std::ranges::find(validOperations, op) == validOperations.end()) { + throw Error("Operation %d not allowed inside derivation", op); + } + } else { + // Operations designed only for the experimental builder-rpc-v0 should never be exposed outside + // derivaitons that use it. + // AddToStoreScanning is still acceptable in ordinary recursive derivations, though. + // Throw the same error we do when using an unknown operation. + if (op == WorkerProto::Op::SubmitOutput + || (op == WorkerProto::Op::AddToStoreScanning && recursive == daemon::RecursiveFlag::NotRecursive)) { + throw Error("invalid operation %1%", op); + } + } + switch (op) { case WorkerProto::Op::IsValidPath: { @@ -565,7 +587,7 @@ static void performOp( if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); - store->buildPaths(drvs, mode); + builder.buildPaths(drvs, mode); logger->stopWork(); conn.to << 1; break; @@ -584,7 +606,7 @@ static void performOp( throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); - auto results = store->buildPathsWithResults(drvs, mode); + auto results = builder.buildPathsWithResults(drvs, mode); logger->stopWork(); WorkerProto::write(*store, wconn, results); @@ -658,12 +680,10 @@ static void performOp( paths. */ assert(drvType.isCA()); - Derivation drv2; - static_cast(drv2) = drv; - drvPath = store->writeDerivation(Derivation{drv2}); + drvPath = store->writeDerivation(drv.unresolve()); } - auto res = store->buildDerivation(drvPath, drv, buildMode); + auto res = builder.buildDerivation(drvPath, drv, buildMode); logger->stopWork(); WorkerProto::write(*store, wconn, res); break; @@ -672,7 +692,7 @@ static void performOp( case WorkerProto::Op::EnsurePath: { auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); - store->ensurePath(path); + builder.ensurePath(path); logger->stopWork(); conn.to << 1; break; @@ -746,14 +766,17 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); } else { auto paths = WorkerProto::Serialise::read(*store, rconn); if (options.action != GCAction::gcDeleteSpecific && paths.empty()) options.pathsToDelete = GCOptions::WholeStore{}; else - options.pathsToDelete = paths; + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = paths, + .deleteReferrers = false, + }; } conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields @@ -761,8 +784,9 @@ static void performOp( readInt(conn.from); readInt(conn.from); - if (options.action == GCAction::gcDeleteDead && std::holds_alternative(options.pathsToDelete) - && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (options.action == GCAction::gcDeleteDead + && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { throw Error( "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); } @@ -809,7 +833,7 @@ static void performOp( // FIXME: use some setting in recursive mode. Will need to use // non-global variables. - if (!recursive) + if (recursive == RecursiveFlag::NotRecursive) clientSettings.apply(trusted); logger->stopWork(); @@ -1022,15 +1046,70 @@ static void performOp( break; } + case WorkerProto::Op::AddToStoreScanning: { + auto name = readString(conn.from); + auto camStr = readString(conn.from); + + experimentalFeatureSettings.require(Xp::DynamicDerivations); + + if (!conn.protoVersion.features.contains(WorkerProto::featureAddToStoreScanning)) + throw Error("Adding to store with scanning was requested, but not supported in negotiated protocol"); + + if (recursive == daemon::RecursiveFlag::NotRecursive) + throw Error( + "AddToStoreScanning only valid within derivation with `builder-rpc-v0` or `recursive-nix` feature"); + + auto & submitStore = require(*store); + + logger->startWork(); + auto pathInfo = [&]() { + // NB: FramedSource must be out of scope before logger->stopWork(); + // FIXME: this means that if there is an error + // half-way through, the client will keep sending + // data, since we haven't sent it the error yet. + auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); + FramedSource source(conn.from); + FileSerialisationMethod dumpMethod = contentAddressMethod.getFileSerialisationMethod(); + return submitStore.addToStoreScanning(source, name, dumpMethod, contentAddressMethod, hashAlgo); + }(); + logger->stopWork(); + + WorkerProto::Serialise::write(*store, wconn, *pathInfo); + break; + } + + case WorkerProto::Op::SubmitOutput: { + experimentalFeatureSettings.require(Xp::DynamicDerivations); + if (recursive != daemon::RecursiveFlag::RecursiveSubmitted) + throw Error("SubmitOutput only valid within derivation with `builder-rpc-v0` feature"); + + auto path = WorkerProto::Serialise::read(*store, rconn); + auto output = WorkerProto::Serialise::read(*store, rconn); + + auto & submitStore = require(*store); + + logger->startWork(); + submitStore.submitOutput(path, output); + logger->stopWork(); + conn.to << 1; + break; + } + default: throw Error("invalid operation %1%", op); } } -void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) +void processConnection( + ref store, + FdSource && from, + FdSink && to, + TrustedFlag trusted, + RecursiveFlag recursive, + std::shared_ptr builder) { #ifndef _WIN32 // TODO need graceful async exit support on Windows? - auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; + auto monitor = (recursive == RecursiveFlag::NotRecursive) ? std::make_unique(from.fd) : nullptr; (void) monitor; // suppress warning ReceiveInterrupts receiveInterrupts; @@ -1045,9 +1124,24 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted }); #endif + if (!builder) + builder = store->getBuilder(); + /* Exchange the greeting. */ + WorkerProto::Version localVersion; + + if (recursive == RecursiveFlag::RecursiveSubmitted) { + localVersion = WorkerProto::builderRpcV0; + } else if (recursive == RecursiveFlag::Recursive) { + localVersion = WorkerProto::latest; + localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); + localVersion.features.insert(std::string{WorkerProto::featureAddToStoreScanning}); + } else { + localVersion = WorkerProto::latest; + } + WorkerProto::BasicServerConnection conn; - conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, WorkerProto::latest); + conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, localVersion); if (conn.protoVersion.number < WorkerProto::minimum.number) throw Error("the Nix client version is too old"); @@ -1055,14 +1149,11 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted conn.to = std::move(to); conn.from = std::move(from); - auto tunnelLogger_ = std::make_unique(conn.to, conn.protoVersion); - auto tunnelLogger = tunnelLogger_.get(); - std::unique_ptr prevLogger_; - auto prevLogger = logger.get(); + auto tunnelLogger = new TunnelLogger(conn.to, conn.protoVersion); + auto prevLogger = logger; // FIXME - if (!recursive) { - prevLogger_ = std::move(logger); - logger = std::move(tunnelLogger_); + if (recursive == RecursiveFlag::NotRecursive) { + logger = tunnelLogger; applyJSONLogger(); } @@ -1108,7 +1199,7 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, conn, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op, *builder); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the diff --git a/src/libstore/darwin/build/darwin-derivation-builder.cc b/src/libstore/darwin/build/darwin-derivation-builder.cc new file mode 100644 index 000000000000..2e05f5858963 --- /dev/null +++ b/src/libstore/darwin/build/darwin-derivation-builder.cc @@ -0,0 +1,313 @@ +#include "derivation-builder-impl.hh" +#include "darwin-derivation-builder.hh" + +#include +#include +#include +#include +#include +#include +#include + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int +sandbox_init_with_parameters(const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf); + +/* Darwin IPC structures and constants */ +#define IPCS_MAGIC 0x00000001 +#define IPCS_SHM_ITER 0x00000002 +#define IPCS_SEM_ITER 0x00000020 +#define IPCS_MSG_ITER 0x00000200 +#define IPCS_SHM_SYSCTL "kern.sysv.ipcs.shm" +#define IPCS_MSG_SYSCTL "kern.sysv.ipcs.msg" +#define IPCS_SEM_SYSCTL "kern.sysv.ipcs.sem" + +struct IpcsCommand +{ + uint32_t ipcs_magic; + uint32_t ipcs_op; + uint32_t ipcs_cursor; + uint32_t ipcs_datalen; + void * ipcs_data; +}; + +namespace nix { + +void DarwinDerivationBuilder::prepareSandbox() +{ + pathsInChroot = getPathsInSandbox(); +} + +void DarwinDerivationBuilder::setUser() +{ + DerivationBuilderImpl::setUser(); + + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useSandbox) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + StringSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + std::filesystem::path cur = i.first; + while (cur != "/") { + cur = cur.parent_path(); + ancestry.insert(cur.native()); + } + } + + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + std::filesystem::path cur = store.storeDir; + while (cur != "/") { + ancestry.insert(cur.native()); + cur = cur.parent_path(); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + pathsInChroot.insert_or_assign(p, ChrootPath{.source = p}); + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be + * configurable */ + if (store.config->getLocalSettings().darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += +#include "sandbox-defaults.sb" + ; + + if (!derivationType.isSandboxed()) + sandboxProfile += +#include "sandbox-network.sb" + ; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + without file-write* allowed, access() incorrectly returns EPERM */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + + // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. + // See https://github.com/NixOS/nix/issues/4119 + // We split our allow groups approximately at half the actual limit, 1 << 16 + const size_t breakpoint = sandboxProfile.length() + (1 << 14); + for (auto & i : pathsInChroot) { + + if (sandboxProfile.length() >= breakpoint) { + debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); + sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; + } + + if (i.first != i.second.source) + throw Error( + "can't map %1% to %2%: mismatched impure paths not supported on Darwin", + PathFmt(i.first), + PathFmt(i.second.source)); + + std::string path = i.first; + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) + continue; + throw SysError("getting attributes of required path '%s", path); + } + if (S_ISDIR(optSt->st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += drvOptions.additionalSandboxProfile; + } else + sandboxProfile += +#include "sandbox-minimal.sb" + ; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different + mechanisms to find temporary directories, so we want to open up a broader place for them to put their files, + if needed. */ + std::filesystem::path globalTmpDir = canonPath(defaultTempDir().native(), true); + + /* They don't like trailing slashes on subpath directives */ + std::string globalTmpDirStr = globalTmpDir.native(); + while (!globalTmpDirStr.empty() && globalTmpDirStr.back() == '/') + globalTmpDirStr.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_NIX_BUILD_TOP"); + sandboxArgs.push_back(tmpDir.native()); + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDirStr); + if (drvOptions.allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + char * sandbox_errbuf = nullptr; + if (sandbox_init_with_parameters( + sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { + writeFull( + STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); + _exit(1); + } + } +} + +void DarwinDerivationBuilder::execBuilder(const Strings & args, const Strings & envStrs) +{ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv.platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv.platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn( + NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +} + +void DarwinDerivationBuilder::cleanupSysVIPCForUser(uid_t uid) +{ + struct IpcsCommand ic; + size_t ic_size = sizeof(ic); + // IPC ids to cleanup + std::vector shm_ids, msg_ids, sem_ids; + + { + struct shmid_ds shm_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SHM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &shm_ds; + ic.ipcs_datalen = sizeof(shm_ds); + + while (true) { + memset(&shm_ds, 0, sizeof(shm_ds)); + + if (sysctlbyname(IPCS_SHM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (shm_ds.shm_perm.uid == uid) { + int shmid = shmget(shm_ds.shm_perm._key, 0, 0); + if (shmid != -1) { + shm_ids.push_back(shmid); + } + } + } + } + + for (auto id : shm_ids) { + if (shmctl(id, IPC_RMID, NULL) == 0) + debug("removed shared memory segment with shmid %d", id); + } + + { + struct msqid_ds msg_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_MSG_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &msg_ds; + ic.ipcs_datalen = sizeof(msg_ds); + + while (true) { + memset(&msg_ds, 0, sizeof(msg_ds)); + + if (sysctlbyname(IPCS_MSG_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (msg_ds.msg_perm.uid == uid) { + int msgid = msgget(msg_ds.msg_perm._key, 0); + if (msgid != -1) { + msg_ids.push_back(msgid); + } + } + } + } + + for (auto id : msg_ids) { + if (msgctl(id, IPC_RMID, NULL) == 0) + debug("removed message queue with msgid %d", id); + } + + { + struct semid_ds sem_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SEM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &sem_ds; + ic.ipcs_datalen = sizeof(sem_ds); + + while (true) { + memset(&sem_ds, 0, sizeof(sem_ds)); + + if (sysctlbyname(IPCS_SEM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (sem_ds.sem_perm.uid == uid) { + int semid = semget(sem_ds.sem_perm._key, 0, 0); + if (semid != -1) { + sem_ids.push_back(semid); + } + } + } + } + + for (auto id : sem_ids) { + if (semctl(id, 0, IPC_RMID) == 0) + debug("removed semaphore with semid %d", id); + } +} + +void DarwinDerivationBuilder::killSandbox(bool getStats) +{ + DerivationBuilderImpl::killSandbox(getStats); + if (buildUser) { + auto uid = buildUser->getUID(); + cleanupSysVIPCForUser(uid); + } +} + +} // namespace nix diff --git a/src/libstore/darwin/build/darwin-derivation-builder.hh b/src/libstore/darwin/build/darwin-derivation-builder.hh new file mode 100644 index 000000000000..260339c5c577 --- /dev/null +++ b/src/libstore/darwin/build/darwin-derivation-builder.hh @@ -0,0 +1,47 @@ +#pragma once + +#include "derivation-builder-impl.hh" + +namespace nix { + +struct DarwinDerivationBuilder : DerivationBuilderImpl +{ + PathsInChroot pathsInChroot; + + /** + * Whether full sandboxing is enabled. Note that macOS builds + * always have *some* sandboxing (see sandbox-minimal.sb). + */ + bool useSandbox; + + DarwinDerivationBuilder( + LocalStore & store, + std::shared_ptr miscMethods, + DerivationBuilderParams params, + bool useSandbox) + : DerivationBuilderImpl(store, miscMethods, std::move(params)) + , useSandbox(useSandbox) + { + } + + void prepareSandbox() override; + + void setUser() override; + + void execBuilder(const Strings & args, const Strings & envStrs) override; + + /** + * Cleans up all System V IPC objects owned by the specified user. + * + * On Darwin, IPC objects (shared memory segments, message queues, and semaphore) + * can persist after the build user's processes are killed, since there are no IPC namespaces + * like on Linux. This can exhaust kernel IPC limits over time. + * + * Uses sysctl to enumerate and remove all IPC objects owned by the given UID. + */ + void cleanupSysVIPCForUser(uid_t uid); + + void killSandbox(bool getStats) override; +}; + +} // namespace nix diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/darwin/build/sandbox-defaults.sb similarity index 100% rename from src/libstore/unix/build/sandbox-defaults.sb rename to src/libstore/darwin/build/sandbox-defaults.sb diff --git a/src/libstore/unix/build/sandbox-minimal.sb b/src/libstore/darwin/build/sandbox-minimal.sb similarity index 100% rename from src/libstore/unix/build/sandbox-minimal.sb rename to src/libstore/darwin/build/sandbox-minimal.sb diff --git a/src/libstore/unix/build/sandbox-network.sb b/src/libstore/darwin/build/sandbox-network.sb similarity index 100% rename from src/libstore/unix/build/sandbox-network.sb rename to src/libstore/darwin/build/sandbox-network.sb diff --git a/src/libstore/darwin/meson.build b/src/libstore/darwin/meson.build new file mode 100644 index 000000000000..72d94baac2be --- /dev/null +++ b/src/libstore/darwin/meson.build @@ -0,0 +1,3 @@ +include_dirs += [ include_directories('build') ] + +sources += files('build/darwin-derivation-builder.cc') diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index b3d4261b5464..36cf9116c833 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -358,7 +358,8 @@ DerivationOptions derivationOptionsFromStructuredAttrs( } template -StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const +template +StringSet DerivationOptions::getRequiredSystemFeatures(const DerivationT & drv) const { // FIXME: cache this? StringSet res; @@ -376,11 +377,20 @@ bool DerivationOptions::substitutesAllowed(const WorkerSettings & workerS } template -bool DerivationOptions::useUidRange(const BasicDerivation & drv) const +template +bool DerivationOptions::useUidRange(const DerivationT & drv) const { return getRequiredSystemFeatures(drv).count("uid-range"); } +// Explicit instantiations for member function templates +template StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation &) const; +template StringSet DerivationOptions::getRequiredSystemFeatures(const Derivation &) const; +template StringSet DerivationOptions::getRequiredSystemFeatures(const Derivation &) const; + +template bool DerivationOptions::useUidRange(const BasicDerivation &) const; +template bool DerivationOptions::useUidRange(const Derivation &) const; + std::optional> tryResolve( const DerivationOptions & drvOptions, fun(ref drvPath, const std::string & outputName)> @@ -536,11 +546,11 @@ template struct DerivationOptions; namespace nlohmann { -using namespace nix; - template -static DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) +static nix::DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) { + using namespace nix; + auto & json = getObject(json_); return { @@ -576,8 +586,10 @@ static DerivationOptions derivationOptionsFromJson(const nlohmann::json } template -static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptions & o) +static void derivationOptionsToJson(nlohmann::json & json, const nix::DerivationOptions & o) { + using namespace nix; + json["outputChecks"] = std::visit( overloaded{ [&](const OutputChecks & checks) { @@ -609,8 +621,10 @@ static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptio } template -static OutputChecks outputChecksFromJson(const nlohmann::json & json_) +static nix::OutputChecks outputChecksFromJson(const nlohmann::json & json_) { + using namespace nix; + auto & json = getObject(json_); return { @@ -625,7 +639,7 @@ static OutputChecks outputChecksFromJson(const nlohmann::json & json_) } template -static void outputChecksToJson(nlohmann::json & json, const OutputChecks & c) +static void outputChecksToJson(nlohmann::json & json, const nix::OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["maxSize"] = c.maxSize; @@ -636,45 +650,51 @@ static void outputChecksToJson(nlohmann::json & json, const OutputChecks json["disallowedRequisites"] = c.disallowedRequisites; } -DerivationOptions adl_serializer>::from_json(const json & json_) +nix::DerivationOptions +adl_serializer>::from_json(const json & json_) { - return derivationOptionsFromJson(json_); + return derivationOptionsFromJson(json_); } -void adl_serializer>::to_json( - json & json, const DerivationOptions & o) +void adl_serializer>::to_json( + json & json, const nix::DerivationOptions & o) { - derivationOptionsToJson(json, o); + derivationOptionsToJson(json, o); } -DerivationOptions adl_serializer>::from_json(const json & json_) +nix::DerivationOptions +adl_serializer>::from_json(const json & json_) { - return derivationOptionsFromJson(json_); + return derivationOptionsFromJson(json_); } -void adl_serializer>::to_json(json & json, const DerivationOptions & o) +void adl_serializer>::to_json( + json & json, const nix::DerivationOptions & o) { - derivationOptionsToJson(json, o); + derivationOptionsToJson(json, o); } -OutputChecks adl_serializer>::from_json(const json & json_) +nix::OutputChecks +adl_serializer>::from_json(const json & json_) { - return outputChecksFromJson(json_); + return outputChecksFromJson(json_); } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +void adl_serializer>::to_json( + json & json, const nix::OutputChecks & c) { - outputChecksToJson(json, c); + outputChecksToJson(json, c); } -OutputChecks adl_serializer>::from_json(const json & json_) +nix::OutputChecks adl_serializer>::from_json(const json & json_) { - return outputChecksFromJson(json_); + return outputChecksFromJson(json_); } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +void adl_serializer>::to_json( + json & json, const nix::OutputChecks & c) { - outputChecksToJson(json, c); + outputChecksToJson(json, c); } } // namespace nlohmann diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 8475532bca33..dd7d69585c6d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,8 +15,6 @@ namespace nix { -using namespace std::literals::string_view_literals; - std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { @@ -99,15 +97,26 @@ bool DerivationType::isImpure() const raw); } -bool BasicDerivation::isBuiltin() const +bool isBuiltin(const BasicDerivation & drv) +{ + return drv.builder.substr(0, 8) == "builtin:"; +} + +template +bool DerivationT::isBuiltin() const { return builder.substr(0, 8) == "builtin:"; } +// Forward declaration of specialization +template<> +std::string DerivationT::unparse( + const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const; + static auto infoForDerivation(const StoreDirConfig & store, const Derivation & drv) { - auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs.map) + auto references = drv.inputs.srcs; + for (auto & i : drv.inputs.drvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -223,6 +232,7 @@ static BackedStringView parseString(StringViewStream & str) size_t start = 0; size_t end = str.remaining.size(); const auto data = str.remaining.data(); + bool foundClose = false; while (start < end) { auto idx = str.remaining.find('"', start); if (idx == std::string_view::npos) { @@ -233,10 +243,13 @@ static BackedStringView parseString(StringViewStream & str) ; if ((idx - pos) % 2 == 0) { // even number of backslashes end = idx; + foundClose = true; break; } start = idx + 1; } + if (!foundClose) + throw FormatError("unterminated string in derivation"); start = 0; const auto content = str.remaining.substr(start, end); @@ -308,6 +321,8 @@ static DerivationOutput parseDerivationOutput( std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { + using namespace std::literals::string_view_literals; + if (!hashAlgoStr.empty()) { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); if (method == ContentAddressMethod::Raw::Text) @@ -387,6 +402,8 @@ enum struct DerivationATermVersion { static DerivedPathMap::ChildNode parseDerivedPathMapNode(const StoreDirConfig & store, StringViewStream & str, DerivationATermVersion version) { + using namespace std::literals::string_view_literals; + DerivedPathMap::ChildNode node; auto parseNonDynamic = [&]() { node.value = parseStrings(str, false); }; @@ -432,8 +449,11 @@ Derivation parseDerivation( std::string_view name, const ExperimentalFeatureSettings & xpSettings) { - Derivation drv; - drv.name = name; + using namespace std::literals::string_view_literals; + + Derivation drv{ + .name = std::string{name}, + }; StringViewStream str{s}; expect(str, 'D'); @@ -477,13 +497,13 @@ Derivation parseDerivation( expect(str, '('); auto drvPath = parsePath(str); expect(str, ','); - drv.inputDrvs.map.insert_or_assign( + drv.inputs.drvs.map.insert_or_assign( store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ')'); } expect(str, ','); - drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + drv.inputs.srcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ','); drv.platform = parseString(str).toOwned(); expect(str, ','); @@ -592,6 +612,8 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt static void unparseDerivedPathMapNode( const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) { + using namespace std::literals::string_view_literals; + s += ','; if (node.childMap.empty()) { printUnquotedStrings(s, node.value.begin(), node.value.end()); @@ -626,15 +648,18 @@ static void unparseDerivedPathMapNode( static bool hasDynamicDrvDep(const Derivation & drv) { return std::find_if( - drv.inputDrvs.map.begin(), - drv.inputDrvs.map.end(), + drv.inputs.drvs.map.begin(), + drv.inputs.drvs.map.end(), [](auto & kv) { return !kv.second.childMap.empty(); }) - != drv.inputDrvs.map.end(); + != drv.inputs.drvs.map.end(); } -std::string Derivation::unparse( +template<> +std::string DerivationT::unparse( const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { + using namespace std::literals::string_view_literals; + std::string s; s.reserve(65536); @@ -720,7 +745,7 @@ std::string Derivation::unparse( s += ')'; } } else { - for (auto & [drvPath, childMap] : inputDrvs.map) { + for (auto & [drvPath, childMap] : inputs.drvs.map) { if (first) first = false; else @@ -733,7 +758,7 @@ std::string Derivation::unparse( } s += "],"sv; - auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow + auto paths = store.printStorePathSet(inputs.srcs); // FIXME: slow printUnquotedStrings(s, paths.begin(), paths.end()); s += ','; @@ -782,6 +807,8 @@ bool isDerivation(std::string_view fileName) std::string outputPathName(std::string_view drvName, OutputNameView outputName) { + using namespace std::literals::string_view_literals; + std::string res{drvName}; if (outputName != "out"sv) { res += '-'; @@ -790,8 +817,11 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName) return res; } -DerivationType BasicDerivation::type() const +template +DerivationType DerivationT::type() const { + using namespace std::literals::string_view_literals; + std::optional floatingHashAlgo; std::optional ty; @@ -925,7 +955,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* For other derivations, replace the inputs paths with recursive calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; - for (auto & [drvPath, node] : drv.inputDrvs.map) { + for (auto & [drvPath, node] : drv.inputs.drvs.map) { /* Need to build and resolve dynamic derivations first */ if (!node.childMap.empty()) { return DrvHashModulo::DeferredDrv{}; @@ -973,7 +1003,8 @@ static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } -StringSet BasicDerivation::outputNames() const +template +StringSet DerivationT::outputNames() const { StringSet names; for (auto & i : outputs) @@ -981,7 +1012,8 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const +template +DerivationOutputsAndOptPaths DerivationT::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; for (auto & [outputName, output] : outputs) @@ -990,7 +1022,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirC return outsAndOptPaths; } -std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) +template +std::string_view DerivationT::nameFromPath(const StorePath & drvPath) { drvPath.requireDerivation(); auto nameWithSuffix = drvPath.name(); @@ -1010,7 +1043,7 @@ Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivati drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = in}); + drv.inputs = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = in}); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -1054,7 +1087,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, i.second.raw); } - CommonProto::write(store, CommonProto::WriteConn{.to = out}, drv.inputSrcs); + CommonProto::write(store, CommonProto::WriteConn{.to = out}, drv.inputs); out << drv.platform << drv.builder << drv.args; auto writeEnv = [&](const StringPairs atermEnv) { @@ -1081,7 +1114,8 @@ std::string hashPlaceholder(const OutputNameView outputName) .to_string(HashFormat::Nix32, false); } -void BasicDerivation::applyRewrites(const StringMap & rewrites) +template +void DerivationT::applyRewrites(const StringMap & rewrites) { if (rewrites.empty()) return; @@ -1111,10 +1145,17 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } +template<> +Derivation DerivationT::unresolve() const +{ + return mapInputs([](const StorePathSet & inputs) -> FullInputs { return {.srcs = inputs, .drvs = {}}; }); +} + +template<> bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ - if (inputDrvs.map.empty()) + if (inputs.drvs.map.empty()) return false; auto drvType = type(); @@ -1139,22 +1180,13 @@ bool Derivation::shouldResolve() const /* Also need to resolve if any inputs are outputs of dynamic derivations. */ bool hasDynamicInputs = std::ranges::any_of( - inputDrvs.map.begin(), inputDrvs.map.end(), [](auto & pair) { return !pair.second.childMap.empty(); }); + inputs.drvs.map.begin(), inputs.drvs.map.end(), [](auto & pair) { return !pair.second.childMap.empty(); }); return typeNeedsResolve || hasDynamicInputs; } -std::optional Derivation::tryResolve(Store & store, Store * evalStore) const -{ - return tryResolve( - store, [&](ref drvPath, const std::string & outputName) -> std::optional { - try { - return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore); - } catch (Error &) { - return std::nullopt; - } - }); -} +template +static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName); static bool tryResolveInput( const StoreDirConfig & store, @@ -1201,20 +1233,49 @@ static bool tryResolveInput( return true; } -std::optional Derivation::tryResolve( +// Forward declaration of specialization +template<> +std::optional DerivationT::tryResolve( + Store & store, + fun(ref drvPath, const std::string & outputName)> + queryResolutionChain) const; + +template<> +std::optional DerivationT::tryResolve(Store & store, Store * evalStore) const +{ + return tryResolve( + store, [&](ref drvPath, const std::string & outputName) -> std::optional { + try { + return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore); + } catch (Error &) { + return std::nullopt; + } + }); +} + +template<> +std::optional DerivationT::tryResolve( Store & store, fun(ref drvPath, const std::string & outputName)> queryResolutionChain) const { - BasicDerivation resolved{*this}; + BasicDerivation resolved{ + .outputs = outputs, + .inputs = inputs.srcs, + .platform = platform, + .builder = builder, + .args = args, + .env = env, + .structuredAttrs = structuredAttrs, + .name = name, + }; - // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputNode] : inputDrvs.map) + for (auto & [inputDrv, inputNode] : inputs.drvs.map) if (!tryResolveInput( store, - resolved.inputSrcs, + resolved.inputs, inputRewrites, nullptr, make_ref(SingleDerivedPath::Opaque{inputDrv}), @@ -1224,11 +1285,9 @@ std::optional Derivation::tryResolve( resolved.applyRewrites(inputRewrites); - Derivation resolved2{std::move(resolved)}; - - resolved2.fillInOutputPaths(store); + processDerivationOutputPaths(store, resolved, resolved.name); - return resolved2; + return resolved; } /** @@ -1260,7 +1319,11 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string auto hashModulo = [&]() -> const auto & { if (!hashModulo_) { // somewhat expensive so we do lazily - hashModulo_ = hashDerivationModulo(store, drv, true); + if constexpr (std::is_same_v, Derivation>) { + hashModulo_ = hashDerivationModulo(store, drv, true); + } else { + hashModulo_ = hashDerivationModulo(store, drv.unresolve(), true); + } } return *hashModulo_; }; @@ -1377,7 +1440,8 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string drv.type(); } -void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +template +void DerivationT::checkInvariants(Store & store, const StorePath & drvPath) const { assert(drvPath.isDerivation()); std::string drvName(drvPath.name()); @@ -1395,16 +1459,25 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const } } +template<> +void BasicDerivation::checkInvariants(Store & store) const +{ + processDerivationOutputPaths(store, *this, name); +} + +template<> void Derivation::checkInvariants(Store & store) const { processDerivationOutputPaths(store, *this, name); } +template<> void Derivation::fillInOutputPaths(Store & store) { processDerivationOutputPaths(store, *this, name); } +template<> Derivation Derivation::parseJsonAndValidate(Store & store, const nlohmann::json & json) { auto drv = static_cast(json); @@ -1423,14 +1496,17 @@ Derivation Derivation::parseJsonAndValidate(Store & store, const nlohmann::json const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); +// Explicit template instantiations +template struct DerivationT; +template struct DerivationT; + } // namespace nix namespace nlohmann { -using namespace nix; - -void adl_serializer::to_json(json & res, const DerivationOutput & o) +void adl_serializer::to_json(json & res, const nix::DerivationOutput & o) { + using namespace nix; res = nlohmann::json::object(); std::visit( overloaded{ @@ -1458,9 +1534,10 @@ void adl_serializer::to_json(json & res, const DerivationOutpu o.raw); } -DerivationOutput -adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +nix::DerivationOutput adl_serializer::from_json( + const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; std::set keys; auto & json = getObject(_json); @@ -1524,15 +1601,42 @@ adl_serializer::from_json(const json & _json, const Experiment } } -static void inputSrcsToJson(json & res, const StorePathSet & inputSrcs) +static void inputsToJson(json & res, const nix::StorePathSet & inputs) { res = nlohmann::json::array(); - for (auto & input : inputSrcs) + for (auto & input : inputs) res.emplace_back(input); } -static void basicDerivationToJson(json & res, const BasicDerivation & d) +static void inputsToJson(json & res, const nix::FullInputs & inputs) { + using namespace nix; + res = nlohmann::json::object(); + + inputsToJson(res["srcs"], inputs.srcs); + + auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + + auto & inputDrvsObj = res["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputs.drvs.map) + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); +} + +template +void adl_serializer>::to_json(json & res, const nix::DerivationT & d) +{ + using namespace nix; res = nlohmann::json::object(); res["name"] = d.name; @@ -1545,6 +1649,8 @@ static void basicDerivationToJson(json & res, const BasicDerivation & d) outputsObj[outputName] = output; } + inputsToJson(res["inputs"], d.inputs); + res["system"] = d.platform; res["builder"] = d.builder; res["args"] = d.args; @@ -1554,148 +1660,119 @@ static void basicDerivationToJson(json & res, const BasicDerivation & d) res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -void adl_serializer::to_json(json & res, const BasicDerivation & d) -{ - basicDerivationToJson(res, d); - - inputSrcsToJson(res["inputs"], d.inputSrcs); -} +template +static Inputs inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings); -void adl_serializer::to_json(json & res, const Derivation & d) +template<> +nix::StorePathSet inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings &) { - basicDerivationToJson(res, d); - - { - auto & inputsObj = res["inputs"]; - inputsObj = nlohmann::json::object(); - - inputSrcsToJson(inputsObj["srcs"], d.inputSrcs); - - auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { - auto value = nlohmann::json::object(); - value["outputs"] = inputNode.value; - { - auto next = nlohmann::json::object(); - for (auto & [outputId, childNode] : inputNode.childMap) - next[outputId] = doInput(childNode); - value["dynamicOutputs"] = std::move(next); - } - return value; - }; - - auto & inputDrvsObj = inputsObj["drvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } -} - -static void inputSrcsFromJson(const json & inputSrcsJson, StorePathSet & inputSrcs) -{ - auto arr = getArray(inputSrcsJson); - for (auto & input : arr) + using namespace nix; + StorePathSet inputSrcs; + for (auto & input : getArray(inputsJson)) inputSrcs.insert(input); + return inputSrcs; } -static void basicDerivationFromJson( - const json::object_t & json, BasicDerivation & res, const ExperimentalFeatureSettings & xpSettings) +template<> +nix::FullInputs +inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings) { - res.name = getString(valueAt(json, "name")); + using namespace nix; - { - auto version = getUnsigned(valueAt(json, "version")); - if (valueAt(json, "version") != expectedJsonVersionDerivation) - throw Error( - "Unsupported derivation JSON format version %d, only format version %d is currently supported.", - version, - expectedJsonVersionDerivation); - } + auto inputsObj = getObject(inputsJson); + FullInputs inputs; try { - auto outputs = getObject(valueAt(json, "outputs")); - for (auto & [outputName, output] : outputs) { - res.outputs.insert_or_assign(outputName, adl_serializer::from_json(output, xpSettings)); - } + for (auto & input : getArray(valueAt(inputsObj, "srcs"))) + inputs.srcs.insert(input); } catch (Error & e) { - e.addTrace({}, "while reading key 'outputs'"); + e.addTrace({}, "while reading key 'srcs'"); throw; } - res.platform = getString(valueAt(json, "system")); - res.builder = getString(valueAt(json, "builder")); - res.args = getStringList(valueAt(json, "args")); - - auto envJson = valueAt(json, "env"); try { - res.env = getStringMap(envJson); + auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { + xpSettings.require( + Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(inputsObj, "drvs"))) + inputs.drvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); + e.addTrace({}, "while reading key 'drvs'"); throw; } - if (auto structuredAttrs = get(json, "structuredAttrs")) - res.structuredAttrs = StructuredAttrs{*structuredAttrs}; + return inputs; } -BasicDerivation -adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +template +nix::DerivationT adl_serializer>::from_json( + const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) { - BasicDerivation res; - auto & json = getObject(_json); - basicDerivationFromJson(json, res, xpSettings); - - try { - inputSrcsFromJson(valueAt(json, "inputs"), res.inputSrcs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputs'"); - throw; - } - - return res; -} + using namespace nix; -Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) -{ - Derivation res; auto & json = getObject(_json); - basicDerivationFromJson(json, res, xpSettings); - - try { - auto inputsObj = getObject(valueAt(json, "inputs")); - - try { - inputSrcsFromJson(valueAt(inputsObj, "srcs"), res.inputSrcs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'srcs'"); - throw; - } - - try { - auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { - auto & json = getObject(_json); - DerivedPathMap::ChildNode node; - node.value = getStringSet(valueAt(json, "outputs")); - auto drvs = getObject(valueAt(json, "dynamicOutputs")); - for (auto & [outputId, childNode] : drvs) { - xpSettings.require( - Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); - node.childMap[outputId] = doInput(childNode); - } - return node; - }; - auto drvs = getObject(valueAt(inputsObj, "drvs")); - for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'drvs'"); - throw; - } - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputs'"); - throw; + { + auto version = getUnsigned(valueAt(json, "version")); + if (version != expectedJsonVersionDerivation) + throw Error( + "Unsupported derivation JSON format version %d, only format version %d is currently supported.", + version, + expectedJsonVersionDerivation); } - return res; + return DerivationT{ + .outputs = + [&] { + DerivationOutputs outputs; + try { + for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) + outputs.insert_or_assign( + outputName, adl_serializer::from_json(output, xpSettings)); + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; + } + return outputs; + }(), + .inputs = + [&] { + try { + return inputsFromJson(valueAt(json, "inputs"), xpSettings); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + }(), + .platform = getString(valueAt(json, "system")), + .builder = getString(valueAt(json, "builder")), + .args = getStringList(valueAt(json, "args")), + .env = + [&] { + try { + return getStringMap(valueAt(json, "env")); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + }(), + .structuredAttrs = [&]() -> std::optional { + if (auto structuredAttrs = get(json, "structuredAttrs")) + return StructuredAttrs{*structuredAttrs}; + return std::nullopt; + }(), + .name = getString(valueAt(json, "name")), + }; } +template struct adl_serializer; +template struct adl_serializer; + } // namespace nlohmann diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 131674aa5595..0aa7c4d34c35 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -224,17 +224,17 @@ const StorePath & DerivedPath::getBaseStorePath() const namespace nlohmann { -void adl_serializer::to_json(json & json, const SingleDerivedPath::Opaque & o) +void adl_serializer::to_json(json & json, const nix::SingleDerivedPath::Opaque & o) { json = o.path; } -SingleDerivedPath::Opaque adl_serializer::from_json(const json & json) +nix::SingleDerivedPath::Opaque adl_serializer::from_json(const json & json) { - return SingleDerivedPath::Opaque{json}; + return {json}; } -void adl_serializer::to_json(json & json, const SingleDerivedPath::Built & sdpb) +void adl_serializer::to_json(json & json, const nix::SingleDerivedPath::Built & sdpb) { json = { {"drvPath", *sdpb.drvPath}, @@ -242,7 +242,7 @@ void adl_serializer::to_json(json & json, const Single }; } -void adl_serializer::to_json(json & json, const DerivedPath::Built & dbp) +void adl_serializer::to_json(json & json, const nix::DerivedPath::Built & dbp) { json = { {"drvPath", *dbp.drvPath}, @@ -250,9 +250,10 @@ void adl_serializer::to_json(json & json, const DerivedPath: }; } -SingleDerivedPath::Built -adl_serializer::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings) +nix::SingleDerivedPath::Built adl_serializer::from_json( + const json & json0, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; auto & json = getObject(json0); auto drvPath = make_ref(static_cast(valueAt(json, "drvPath"))); drvRequireExperiment(*drvPath, xpSettings); @@ -262,9 +263,10 @@ adl_serializer::from_json(const json & json0, const Ex }; } -DerivedPath::Built -adl_serializer::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings) +nix::DerivedPath::Built adl_serializer::from_json( + const json & json0, const nix::ExperimentalFeatureSettings & xpSettings) { + using namespace nix; auto & json = getObject(json0); auto drvPath = make_ref(static_cast(valueAt(json, "drvPath"))); drvRequireExperiment(*drvPath, xpSettings); @@ -274,31 +276,32 @@ adl_serializer::from_json(const json & json0, const Experime }; } -void adl_serializer::to_json(json & json, const SingleDerivedPath & sdp) +void adl_serializer::to_json(json & json, const nix::SingleDerivedPath & sdp) { std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); } -void adl_serializer::to_json(json & json, const DerivedPath & sdp) +void adl_serializer::to_json(json & json, const nix::DerivedPath & sdp) { std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); } -SingleDerivedPath -adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) +nix::SingleDerivedPath adl_serializer::from_json( + const json & json, const nix::ExperimentalFeatureSettings & xpSettings) { if (json.is_string()) - return static_cast(json); + return static_cast(json); else - return adl_serializer::from_json(json, xpSettings); + return adl_serializer::from_json(json, xpSettings); } -DerivedPath adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) +nix::DerivedPath +adl_serializer::from_json(const json & json, const nix::ExperimentalFeatureSettings & xpSettings) { if (json.is_string()) - return static_cast(json); + return static_cast(json); else - return adl_serializer::from_json(json, xpSettings); + return adl_serializer::from_json(json, xpSettings); } } // namespace nlohmann diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 73ed2b74a7b6..4a73e9daf3ed 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -53,11 +53,11 @@ DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( namespace nlohmann { -using namespace nix; - template -DrvRef adl_serializer>::from_json(const json & json) +nix::DrvRef adl_serializer>::from_json(const json & json) { + using namespace nix; + // OutputName case: { "drvPath": "self", "output": } if (json.type() == nlohmann::json::value_t::object) { auto & obj = getObject(json); @@ -74,8 +74,10 @@ DrvRef adl_serializer>::from_json(const json & json) } template -void adl_serializer>::to_json(json & json, const DrvRef & ref) +void adl_serializer>::to_json(json & json, const nix::DrvRef & ref) { + using namespace nix; + std::visit( overloaded{ [&](const OutputName & outputName) { @@ -88,7 +90,7 @@ void adl_serializer>::to_json(json & json, const DrvRef & ref ref); } -template struct adl_serializer>; -template struct adl_serializer>; +template struct adl_serializer>; +template struct adl_serializer>; } // namespace nlohmann diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 052ec9b1283e..d03b7f6fdb72 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -10,6 +10,10 @@ namespace nix { +void DummyStoreConfig::anchor() {} + +void DummyStore::anchor() {} + std::string DummyStoreConfig::doc() { return @@ -31,6 +35,8 @@ namespace { class WholeStoreViewAccessor : public SourceAccessor { + void anchor() override {}; + using BaseName = std::string; /** @@ -63,6 +69,8 @@ class WholeStoreViewAccessor : public SourceAccessor }); if (!res) + /* The accessor is truly empty, i.e. without any file at root so + any subsequent operation with it will fail. */ res = &emptyAccessor; return callback(*res, path); @@ -101,6 +109,7 @@ class WholeStoreViewAccessor : public SourceAccessor DirEntries readDirectory(const CanonPath & path) override { + /* FIXME: Special-case the root directory to read the whole store, not just an empty root. */ return callWithAccessorForPath( path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); } @@ -126,6 +135,10 @@ bool DummyStoreConfig::getReadOnly() const struct DummyStoreImpl : DummyStore { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; /** @@ -378,6 +391,8 @@ struct DummyStoreImpl : DummyStore } }; +void DummyStoreImpl::anchor() {} + ref DummyStore::Config::openDummyStore() const { return make_ref(ref{shared_from_this()}); @@ -389,10 +404,9 @@ static RegisterStoreImplementation regDummyStore; namespace nlohmann { -using namespace nix; - -DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) +nix::DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); return DummyStore::PathInfoAndContents{ .info = valueAt(obj, "info"), @@ -400,7 +414,8 @@ DummyStore::PathInfoAndContents adl_serializer: }; } -void adl_serializer::to_json(json & json, const DummyStore::PathInfoAndContents & val) +void adl_serializer::to_json( + json & json, const nix::DummyStore::PathInfoAndContents & val) { json = { {"info", val.info}, @@ -408,8 +423,9 @@ void adl_serializer::to_json(json & json, const }; } -ref adl_serializer>::from_json(const json & json) +nix::ref adl_serializer>::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); auto cfg = make_ref(DummyStore::Config::Params{}); cfg->storeDir_.set(getString(valueAt(obj, "store"))); @@ -417,15 +433,16 @@ ref adl_serializer>::from_json(const j return cfg; } -void adl_serializer::to_json(json & json, const DummyStoreConfig & val) +void adl_serializer::to_json(json & json, const nix::DummyStoreConfig & val) { json = { {"store", val.storeDir}, }; } -ref adl_serializer>::from_json(const json & json) +nix::ref adl_serializer>::from_json(const json & json) { + using namespace nix; auto & obj = getObject(json); ref res = adl_serializer>::from_json(valueAt(obj, "config"))->openDummyStore(); for (auto & [k, v] : getObject(valueAt(obj, "contents"))) @@ -442,8 +459,9 @@ ref adl_serializer>::from_json(const json & json) return res; } -void adl_serializer::to_json(json & json, const DummyStore & val) +void adl_serializer::to_json(json & json, const nix::DummyStore & val) { + using namespace nix; json = { {"config", *val.config}, {"contents", diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index b1c61626c8c7..c0b76158a822 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -37,9 +37,8 @@ static void exportPath(Store & store, const StorePath & path, Sink & sink) void exportPaths(Store & store, const StorePathSet & paths, Sink & sink) { auto sorted = store.topoSortPaths(paths); - std::reverse(sorted.begin(), sorted.end()); - for (auto & path : sorted) { + for (auto & path : sorted | std::views::reverse) { sink << 1; exportPath(store, path, sink); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 85bc650dc8d4..6e9f2256eaa0 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -96,6 +97,8 @@ std::optional FileTransferSettings::getDefaultSSLCertFile return std::nullopt; } +void FileTransferSettings::anchor() {} + FileTransferSettings::FileTransferSettings() { std::optional sslOverride = @@ -113,6 +116,8 @@ FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); +FileTransfer::~FileTransfer() {} + namespace { using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>; @@ -129,12 +134,21 @@ struct curlMultiError final : CloneableError } }; +/* Check if the linked libcurl was built with HTTP3 support. */ +bool curlSupportsHttp3() +{ + const auto * info = ::curl_version_info(CURLVERSION_NOW); + return info && (info->features & CURL_VERSION_HTTP3); +} + } // namespace struct curlFileTransfer : public FileTransfer { const FileTransferSettings & settings; + const bool http3Supported = curlSupportsHttp3(); + curlMulti curlm; std::random_device rd; @@ -280,8 +294,7 @@ struct curlFileTransfer : public FileTransfer } try { if (!done && enqueued) - fail(FileTransferError( - Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri)); + failInterruptedOrCancelled(); } catch (...) { ignoreExceptionInDestructor(); } @@ -297,7 +310,7 @@ struct curlFileTransfer : public FileTransfer /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ - e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); + e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.displayUri()); } catch (...) { /* Can't add more context to the error. */ } @@ -310,6 +323,18 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward(e))); } + void failInterruptedOrCancelled() + { + HintFmt fmt("%s of '%s' was interrupted", Uncolored(request.noun()), request.displayUri()); + + /* Technically, we don't really have per-transfer cancellation currently, + but it's nice to distinguish between the two in the future. */ + if (getInterrupted()) + fail(nix::Interrupted(std::move(fmt))); + else + fail(nix::Cancelled(std::move(fmt))); + } + LambdaSink finalSink; std::optional errorSink; @@ -361,7 +386,7 @@ struct curlFileTransfer : public FileTransfer try { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.displayUri(), trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); if (std::smatch match; std::regex_match(line, match, statusLine)) { @@ -450,8 +475,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), - Logger::Fields{request.uri.to_string()}, + fmt("%s '%s'", request.verb(/*continuous=*/true), request.displayUri()), + Logger::Fields{request.displayUri()}, request.parentAct); // Reset the start time to when we actually started the download. startTime = std::chrono::steady_clock::now(); @@ -590,7 +615,10 @@ struct curlFileTransfer : public FileTransfer : "")) .c_str()); curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); - if (fileTransfer.settings.enableHttp2) + /* Enable HTTP3 only on user config and linked libcurl have support, o.w. fall back. */ + if (fileTransfer.settings.enableHttp3 && fileTransfer.http3Supported) + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_3); + else if (fileTransfer.settings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -651,6 +679,14 @@ struct curlFileTransfer : public FileTransfer #endif curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransfer.settings.connectTimeout.get()); + /* Enable TCP keepalive to detect dead connections and server closures. + Probes every 30s to catch network failures and idle timeouts early. */ + curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L); + curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 30L); + curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 30L); + /* Don't reuse idle connections older than 90s. */ + curl_easy_setopt(req, CURLOPT_MAXAGE_CONN, 90L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransfer.settings.stalledDownloadTimeout.get()); @@ -716,7 +752,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", Uncolored(request.noun()), - request.uri, + request.displayUri(), code, httpStatus, result.bodySize, @@ -752,14 +788,47 @@ struct curlFileTransfer : public FileTransfer // We treat most errors as transient, but won't retry when hopeless Error err = Transient; - if (httpStatus == HttpStatus::NotFound || httpStatus == HttpStatus::Gone + // S3 returns certain retryable errors as HTTP 400/500/503 with XML error codes. + // These take precedence over the generic HTTP status handling below. + // Only parse the response body on status codes where S3 XML errors can appear. + static constexpr std::array s3RetryableErrors{{ + "IncompleteBody", // HTTP 400 - network issue + "InternalError", // HTTP 500 - S3 internal failure + "InternalFailure", // HTTP 500 - alias for InternalError + "InternalServerError", // HTTP 500 - alias for InternalError + "RequestExpired", // HTTP 400 - clock skew / slow upload + "RequestTimeout", // HTTP 400 - stale connection reuse + "RequestTimeTooSkewed", // HTTP 403 - clock drift + "RequestThrottled", // HTTP 400 - throttling variant + "SlowDown", // HTTP 503 - throttling + "ServiceUnavailable", // HTTP 503 - temporary unavailability + "Throttling", // HTTP 400 - throttling variant + "ThrottledException", // HTTP 400 - throttling variant + }}; + // S3 error responses have the form ....... + // Require the root to avoid matching unrelated XML with a element. + static std::regex s3ErrorCodeRegex("[^]*([^<]+)"); + std::smatch s3Match; + bool isS3XmlStatus = httpStatus == 400 || httpStatus == 403 || httpStatus == 500 || httpStatus == 503; + auto s3ErrorCode = + (isS3XmlStatus && errorSink && std::regex_search(errorSink->s, s3Match, s3ErrorCodeRegex)) + ? s3Match[1].str() + : ""; + + if (std::find(s3RetryableErrors.begin(), s3RetryableErrors.end(), s3ErrorCode) + != s3RetryableErrors.end()) { + debug("S3 error '%s', will retry", s3ErrorCode); + } else if ( + httpStatus == HttpStatus::NotFound || httpStatus == HttpStatus::Gone || code == CURLE_FILE_COULDNT_READ_FILE) { // The file is definitely not there err = NotFound; - } else if ( - httpStatus == HttpStatus::Unauthorized || httpStatus == HttpStatus::Forbidden - || httpStatus == HttpStatus::ProxyAuthRequired) { - // Don't retry on authentication/authorization failures + } else if (httpStatus == HttpStatus::Unauthorized || httpStatus == HttpStatus::ProxyAuthRequired) { + err = Unauthorized; + } else if (httpStatus == HttpStatus::Forbidden) { + // Don't retry on authentication/authorization failures. + // Note: the only reason we treat this differently from 401/407 is S3 returns 403 if a file doesn't + // exist and the bucket is unlistable. err = Forbidden; } else if ( httpStatus >= 400 && httpStatus < 500 && httpStatus != HttpStatus::RequestTimeout @@ -810,19 +879,20 @@ struct curlFileTransfer : public FileTransfer std::optional response; if (errorSink) response = std::move(errorSink->s); - auto exc = code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError( - Interrupted, - std::move(response), - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.uri) - : httpStatus != 0 + + /* TODO: Also support per-transfer cancellations. */ + if (code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()) { + failInterruptedOrCancelled(); + return; + } + + auto exc = httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", Uncolored(request.verb()), - request.uri, + request.displayUri(), httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError( @@ -830,7 +900,7 @@ struct curlFileTransfer : public FileTransfer std::move(response), "unable to %s '%s': %s (%d) %s", Uncolored(request.verb()), - request.uri, + request.displayUri(), curl_easy_strerror(code), code, errbuf); @@ -927,6 +997,8 @@ struct curlFileTransfer : public FileTransfer private: bool quitting = false; public: + bool work = false; + void quit() { quitting = true; @@ -969,25 +1041,19 @@ struct curlFileTransfer : public FileTransfer workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~curlFileTransfer() - { - try { - stopWorkerThread(); - } catch (...) { - ignoreExceptionInDestructor(); - } - workerThread.join(); - } + ~curlFileTransfer() override; void stopWorkerThread() { /* Signal the worker thread to exit. */ - state_.lock()->quit(); - wakeupMulti(); + auto state(state_.lock()); + state->quit(); + wakeupMulti(*state); } - void wakeupMulti() + void wakeupMulti(State & state) { + state.work = true; if (auto ec = ::curl_multi_wakeup(curlm.get())) throw curlMultiError(ec); } @@ -1037,25 +1103,12 @@ struct curlFileTransfer : public FileTransfer } } - /* Wait for activity, including wakeup events. */ - long maxSleepTimeMs = items.empty() ? 10000 : 100; - auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() - ? std::max( - 0, - (int) std::chrono::duration_cast( - nextWakeup - std::chrono::steady_clock::now()) - .count()) - : maxSleepTimeMs; - - int numfds = 0; - mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw curlMultiError(mc); - nextWakeup = std::chrono::steady_clock::time_point(); std::vector> incoming; + std::vector> unpause; auto now = std::chrono::steady_clock::now(); + bool haveWork; { auto state(state_.lock()); @@ -1077,25 +1130,23 @@ struct curlFileTransfer : public FileTransfer break; } } + unpause = std::exchange(state->unpause, {}); quit = state->isQuitting(); + haveWork = std::exchange(state->work, false); } for (auto & item : incoming) { - debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.displayUri()); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; items[item->req] = item; } - /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ - auto unpause = [&]() { - auto state(state_.lock()); - auto res = state->unpause; - state->unpause.clear(); - return res; - }(); + if (quit) + break; + /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ for (auto & item : unpause) { /* The transfer might have completed (failed) between it getting enqueued for unpause and by the time the worker thread picked @@ -1105,6 +1156,26 @@ struct curlFileTransfer : public FileTransfer continue; static_cast(*ptr).unpause(); } + + /* Wait for activity, including wakeup events. */ + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + (int) std::chrono::duration_cast( + nextWakeup - std::chrono::steady_clock::now()) + .count()) + : maxSleepTimeMs; + + /* Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 (8.21), + curl_multi_perform seems to swallow queued up events ¯\_(ツ)_/¯. */ + if (haveWork) + sleepTimeMs = 0; + + int numfds = 0; + mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw curlMultiError(mc); } debug("download thread shutting down"); @@ -1133,7 +1204,7 @@ struct curlFileTransfer : public FileTransfer { if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" && item->request.uri.scheme() != "s3") - throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); + throw nix::Error("uploading to '%s' is not supported", item->request.displayUri()); { auto state(state_.lock()); @@ -1141,9 +1212,9 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); item->enqueued = true; /* Now any exceptions should be reported via the callback. */ + wakeupMulti(*state); } - wakeupMulti(); return ItemHandle(item.get_ptr()); } @@ -1163,7 +1234,7 @@ struct curlFileTransfer : public FileTransfer { auto state(state_.lock()); state->unpause.push_back(std::move(item)); - wakeupMulti(); + wakeupMulti(*state); } void unpauseTransfer(ItemHandle handle) override @@ -1174,19 +1245,31 @@ struct curlFileTransfer : public FileTransfer } }; +curlFileTransfer::~curlFileTransfer() +{ + try { + stopWorkerThread(); + } catch (...) { + ignoreExceptionInDestructor(); + } + workerThread.join(); +} + ref makeCurlFileTransfer(const FileTransferSettings & settings = fileTransferSettings) { return make_ref(settings); } +static auto * const _fileTransfer = new Sync>; + ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer->lock()); - if (fileTransfer->state_.lock()->isQuitting()) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) + *fileTransfer = makeCurlFileTransfer().get_ptr(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer(const FileTransferSettings & settings) @@ -1194,6 +1277,20 @@ ref makeFileTransfer(const FileTransferSettings & settings) return makeCurlFileTransfer(settings); } +std::string FileTransferRequest::displayUri() const +{ + try { + auto parsed = uri.parsed(); + if (parsed.authority && parsed.authority->user) { + parsed.authority->user.reset(); + parsed.authority->password.reset(); + return parsed.to_string(); + } + } catch (BadURL &) { + } + return uri.to_string(); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1270,7 +1367,7 @@ void FileTransfer::download( bool paused = false; std::exception_ptr exc; std::string data; - std::condition_variable avail, request; + std::condition_variable avail; }; auto _state = std::make_shared>(); @@ -1280,10 +1377,9 @@ void FileTransfer::download( Finally finally([&]() { auto state(_state->lock()); state->quit = true; - state->request.notify_one(); }); - request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { + request.dataCallback = [_state, uri = request.displayUri()](std::string_view data) -> PauseTransfer { auto state(_state->lock()); if (state->quit) @@ -1326,7 +1422,6 @@ void FileTransfer::download( state->exc = std::current_exception(); } state->avail.notify_one(); - state->request.notify_one(); }}); while (true) { @@ -1360,8 +1455,6 @@ void FileTransfer::download( chunk = std::move(state->data); /* Reset state->data after the move, since we check data.empty() */ state->data = ""; - - state->request.notify_one(); } /* Flush the data to the sink and wake up the download thread @@ -1372,6 +1465,8 @@ void FileTransfer::download( } } +void FileTransferError::anchor() {} + template FileTransferError::FileTransferError( FileTransfer::Error error, std::optional response, const Args &... args) diff --git a/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh b/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh new file mode 100644 index 000000000000..7ff6d1fb4615 --- /dev/null +++ b/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "chroot-derivation-builder.hh" +#include "freebsd-derivation-builder.hh" + +#include "nix/util/freebsd-jail.hh" + +namespace nix { + +struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivationBuilder +{ + std::shared_ptr autoDelJail = std::make_shared(); + + ChrootFreeBSDDerivationBuilder( + LocalStore & store, std::shared_ptr miscMethods, DerivationBuilderParams params) + : DerivationBuilderImpl{store, miscMethods, params} + , ChrootDerivationBuilder{store, miscMethods, params} + , FreeBSDDerivationBuilder{store, miscMethods, params} + { + } + + virtual void cleanupBuild(bool force) override; + + void prepareSandbox() override; + + void startChild() override; + + void enterChroot() override; + + void addDependencyImpl(const StorePath & path) override; +}; + +} // namespace nix diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.cc b/src/libstore/freebsd/build/freebsd-derivation-builder.cc new file mode 100644 index 000000000000..68ccdafa1ae5 --- /dev/null +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.cc @@ -0,0 +1,484 @@ +#include "derivation-builder-impl.hh" +#include "freebsd-derivation-builder.hh" +#include "chroot-derivation-builder.hh" +#include "chroot-freebsd-derivation-builder.hh" + +#include "nix/util/freebsd-jail.hh" +#include "nix/util/util.hh" +#include "nix/store/filetransfer.hh" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + +FreeBSDDerivationBuilder::~FreeBSDDerivationBuilder() {} + +namespace { + +struct PasswordEntry +{ + std::string name; + uid_t uid; + gid_t gid; + std::string description; + std::filesystem::path home; + std::filesystem::path shell; +}; + +using UniqueDB = std::unique_ptr<::DB, decltype([](::DB * db) { + if (db) + (db->close)(db); + })>; + +// Database open flags from FreeBSD, in case they're necessary for compatibility +static constexpr HASHINFO dbFlags = { + .bsize = 4096, + .ffactor = 32, + .nelem = 256, + .cachesize = 2 * 1024 * 1024, + .hash = nullptr, + .lorder = BIG_ENDIAN, +}; + +// Password database version +// Version 4 has been current since 2003 +static const uint8_t dbVersion = 4; + +static void serializeString(std::vector & buf, std::string const & str) +{ + buf.insert(buf.end(), str.begin(), str.end()); + buf.push_back(0); +} + +static void serializeInt(std::vector & buf, uint32_t num) +{ + // Always big endian + buf.push_back((num >> 24) & 0xff); + buf.push_back((num >> 16) & 0xff); + buf.push_back((num >> 8) & 0xff); + buf.push_back((num >> 0) & 0xff); +} + +static std::vector byNameKey(std::string const & name) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; + buf.reserve(1 + name.size()); + // We can't use serializeString since that's null terminated + buf.insert(buf.end(), name.begin(), name.end()); + + return buf; +} + +static std::vector byNumKey(uint32_t num) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNUM, dbVersion)}; + serializeInt(buf, num); + + return buf; +} + +static std::vector byUidKey(uid_t uid) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYUID, dbVersion)}; + serializeInt(buf, uid); + + return buf; +} + +static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) +{ + auto db = + UniqueDB(::dbopen((chrootRootDir / "etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &dbFlags)); + + if (!db) + throw SysError("could not create password database"); + + auto dbInsert = [&db](std::vector keyBuf, std::vector & valueBuf) { + DBT key = {keyBuf.data(), keyBuf.size()}; + DBT value = {valueBuf.data(), valueBuf.size()}; + + if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { + throw SysError("could not write to password database"); + } + }; + + // Annoyingly DBT doesn't have const pointers so we need this whole shuffle + std::string versionKeyStr(_PWD_VERSION_KEY); + std::vector versionKey(versionKeyStr.begin(), versionKeyStr.end()); + std::vector versionValue{dbVersion}; + dbInsert(versionKey, versionValue); + + for (const auto & [i, user] : enumerate(users)) { + // flags for non-empty fields + uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; + + std::vector buf; + serializeString(buf, user.name); + // pw_password is always "*" in the insecure database + serializeString(buf, std::string("*")); + serializeInt(buf, user.uid); + serializeInt(buf, user.gid); + // pw_change = 0 means no requirement to change password + serializeInt(buf, 0); + // pw_class is empty since we don't make a class database + serializeString(buf, std::string("")); + serializeString(buf, user.description); + serializeString(buf, user.home); + serializeString(buf, user.shell); + // pw_expire = 0 means password does not expire + serializeInt(buf, 0); + serializeInt(buf, fields); + + dbInsert(byNameKey(user.name), buf); + // _PW_KEYBYNUM is 1-indexed + dbInsert(byNumKey(i + 1), buf); + dbInsert(byUidKey(user.uid), buf); + } + + // FreeBSD libc doesn't use /etc/passwd, but some software might + std::string passwdContent; + for (const auto & user : users) { + passwdContent.append( + fmt("%s:*:%d:%d:%s:%s:%s\n", + user.name, + user.uid, + user.gid, + user.description, + user.home.native(), + user.shell.native())); + } + + writeFile(chrootRootDir / "etc/passwd", passwdContent); + + // No need to make /etc/master.passwd or /etc/spwd.db, + // our build user wouldn't be able to read them anyway +} + +} // namespace + +template +struct iovec iovFromMutableBuffer(std::array & array) +{ + return { + .iov_base = static_cast(array.data()), + .iov_len = N, + }; +} + +template +struct iovec iovFromStaticSizedString(const char (&array)[N]) +{ + return { + .iov_base = const_cast(static_cast(array)), + .iov_len = N, + }; +} + +struct iovec iovFromDynamicSizeString(const std::string & s) +{ + return { + .iov_base = const_cast(static_cast(s.c_str())), + .iov_len = s.length() + 1, + }; +} + +void ChrootFreeBSDDerivationBuilder::cleanupBuild(bool force) +{ + autoDelJail->remove(); + ChrootDerivationBuilder::cleanupBuild(force); +} + +void ChrootFreeBSDDerivationBuilder::prepareSandbox() +{ + ChrootDerivationBuilder::prepareSandbox(); + + std::vector users{ + { + .name = "root", + .uid = 0, + .gid = 0, + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nixbld", + .uid = buildUser->getUID(), + .gid = sandboxGid(), + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nobody", + .uid = 65534, + .gid = 65534, + .description = "Nobody", + .home = "/", + .shell = "/noshell", + }, + }; + + createPasswordFiles(chrootRootDir, users); + + // FreeBSD doesn't have a group database, just write a text file + writeFile( + chrootRootDir / "etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", + sandboxGid())); + + // Linux waits until after entering the child to start mounting so it doesn't + // pollute the root mount namespace. + // FreeBSD doesn't have mount namespaces, so there's no reason to wait. + + auto devpath = chrootRootDir / "dev"; + createDir(devpath, 0555); + createDir(chrootRootDir / "bin", 0555); + + std::array errmsg{}; + std::array<::iovec, 8> iov = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("devfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(devpath.native()), + iovFromStaticSizedString("ruleset"), + iovFromStaticSizedString("4"), + iovFromStaticSizedString("errmsg"), + iovFromMutableBuffer(errmsg), + }; + + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount jail /dev: %1%", std::string_view(errmsg.data())); + + autoDelJail->childrenMounts.emplace_back(devpath); + + for (const auto & [target, chrootPath] : pathsInChroot) { + std::filesystem::path path = chrootRootDir / target.relative_path(); + + auto maybeSt = maybeLstat(chrootPath.source); + if (!maybeSt) { + if (chrootPath.optional) + continue; /* Skip mounting this path. */ + else + throw SysError("getting attributes of path %1%", PathFmt(chrootPath.source)); + } + + /* Mount points must exist and be the right type. */ + if (S_ISDIR(maybeSt->st_mode)) { + createDirs(path); + } else if (S_ISLNK(maybeSt->st_mode)) { + createDirs(path.parent_path()); + copyFile(chrootPath.source, path, /*andDelete=*/false, /*contents=*/false); + continue; + } else { + createDirs(path.parent_path()); + writeFile(path, ""); + } + + std::array<::iovec, 8> iov = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("nullfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(path.native()), + iovFromStaticSizedString("target"), + iovFromDynamicSizeString(chrootPath.source.native()), + iovFromStaticSizedString("errmsg"), + iovFromMutableBuffer(errmsg), + }; + + debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); + + int flags = 0; + if (store.isInStore(target.native())) + /* While we are at it, enforce invariants about store paths. Anything located at the "logical" store + location must be readonly (file permission canonicalisation enforces this on the host filesystem). + Also the store must never contain setuid binaries for the same reason. This is just defense-in-depth. */ + flags = MNT_RDONLY | MNT_NOSUID; + + if (nmount(iov.data(), iov.size(), flags) < 0) + throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); + + autoDelJail->childrenMounts.emplace_back(path); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType.isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir / "etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (std::filesystem::path path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"}) { + if (pathExists(path)) { + // This means if your network config changes during a FOD build, + // the DNS in the sandbox will be wrong. However, this is pretty unlikely + // to actually be a problem, because FODs are generally pretty fast, + // and machines with often-changing network configurations probably + // want to run resolved or some other local resolver anyway. + // + // There's also just no simple way to do this correctly, you have to manually + // inotify watch the files for changes on the outside and update the sandbox + // while the build is running (or at least that's what Flatpak does). + // + // I also just generally feel icky about modifying sandbox state under a build, + // even though it really shouldn't be a big deal. -K900 + copyFile(path, chrootRootDir / path.relative_path(), false, true); + } + } + + if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { + // For the same reasons as above, copy the CA certificates file too. + // It should be even less likely to change during the build than resolv.conf. + createDirs(chrootRootDir / "etc/ssl/certs"); + copyFile( + fileTransferSettings.caFile.get().value(), + chrootRootDir / "etc/ssl/certs/ca-certificates.crt", + false, + true); + } + } +} + +void ChrootFreeBSDDerivationBuilder::startChild() +{ + int jid; + + RunChildArgs args{ +#if NIX_WITH_AWS_AUTH + .awsCredentials = preResolveAwsCredentials(), +#endif + }; + + if (derivationType.isSandboxed()) { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + // TODO: Make our own ruleset + "vnet", + "new", + nullptr); + if (jid < 0) { + throw SysError("failed to create jail (isolated network): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + + // Everything from here to the end of the block is setting up the network + // code adapted from freebsd/sbin/ifconfig/af_inet.c, in_exec_nl + Pid helper = startProcess([&]() { + unix::closeExtraFDs(); + enterChroot(); + + struct snl_state ss = {}; + if (!snl_init(&ss, NETLINK_ROUTE)) { + throw SysError("Failed to init netlink connection"); + } + + struct snl_writer nw = {}; + snl_init_writer(&ss, &nw); + struct nlmsghdr * hdr = snl_create_msg_request(&nw, NL_RTM_NEWADDR); + struct ifaddrmsg * ifahdr = snl_reserve_msg_object(&nw, struct ifaddrmsg); + + ifahdr->ifa_family = AF_INET; + ifahdr->ifa_prefixlen = 8; + ifahdr->ifa_index = if_nametoindex("lo0"); + snl_add_msg_attr_ip4(&nw, IFA_LOCAL, (const struct in_addr *) "\x7f\x00\x00\x01"); + + int off = snl_add_msg_attr_nested(&nw, IFA_FREEBSD); + snl_add_msg_attr_u32(&nw, IFAF_FLAGS, IFF_LOOPBACK | IFF_UP); + snl_end_attr_nested(&nw, off); + + if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { + snl_free(&ss); + throw SysError("failed to sendoff netlink message"); + } + + struct snl_errmsg_data e = {}; + snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); + if (e.error_str != nullptr) { + snl_free(&ss); + throw SysError("failed to configure loopback interface: %1%", e.error_str); + } + snl_free(&ss); + _exit(0); + }); + + /* TODO: Capture the error from the helper? */ + if (auto status = helper.wait(); !statusOk(status)) { + throw Error("failed to configure loopback address: %s", statusToString(status)); + } + } else { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + // 4 is the most restrictive devfs ruleset that meets our needs + // which is found in the default installation. Trying to add + // another one is a huge pain... + "devfs_ruleset", + "4", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + "ip4", + "inherit", + "ip6", + "inherit", + "allow.raw_sockets", + "true", + nullptr); + if (jid < 0) { + throw SysError("failed to create jail (networked): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + } + + pid = startProcess([&]() { + openSlave(); + runChild(args); + }); +} + +void ChrootFreeBSDDerivationBuilder::enterChroot() +{ + /* Close all other file descriptors. This must happen before + jail_attach for FreeBSD. */ + unix::closeExtraFDs(); + + if (jail_attach(autoDelJail->jid) < 0) { + throw SysError("failed to attach to jail"); + } +} + +void ChrootFreeBSDDerivationBuilder::addDependencyImpl(const StorePath & path) +{ + throw UnimplementedError( + "adding store path '%s' to the sandbox is not implemented (recursive-nix)", store.printStorePath(path)); +} + +} // namespace nix diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.hh b/src/libstore/freebsd/build/freebsd-derivation-builder.hh new file mode 100644 index 000000000000..ce607db3982c --- /dev/null +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "derivation-builder-impl.hh" + +namespace nix { + +struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl +{ + using DerivationBuilderImpl::DerivationBuilderImpl; + + FreeBSDDerivationBuilder(FreeBSDDerivationBuilder &&) = delete; + FreeBSDDerivationBuilder(const FreeBSDDerivationBuilder &) = delete; + FreeBSDDerivationBuilder & operator=(FreeBSDDerivationBuilder &&) = delete; + FreeBSDDerivationBuilder & operator=(const FreeBSDDerivationBuilder &) = delete; + /* To appease Wweak-vtables. */ + ~FreeBSDDerivationBuilder() override; +}; + +} // namespace nix diff --git a/src/libstore/freebsd/meson.build b/src/libstore/freebsd/meson.build new file mode 100644 index 000000000000..bac3949b965f --- /dev/null +++ b/src/libstore/freebsd/meson.build @@ -0,0 +1,3 @@ +include_dirs += [ include_directories('build') ] + +sources += files('build/freebsd-derivation-builder.cc') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 835d06864671..8e997c8e2471 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -37,6 +37,10 @@ namespace nix { +void LocalSettings::anchor() {} + +void GCSettings::anchor() {} + static std::string gcSocketPath = "gc-socket/socket"; static std::string gcRootsDir = "gcroots"; @@ -170,8 +174,6 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) } auto path = i.path(); - pid_t pid = std::stoi(name); - debug("reading temporary root file %1%", PathFmt(path)); AutoCloseFD fd(toDescriptor(open( path.string().c_str(), @@ -206,7 +208,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != std::string::npos) { auto root = std::string_view(contents).substr(pos, end - pos); debug("got temporary root '%s'", root); - tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%s}", name)); pos = end + 1; } } @@ -357,14 +359,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) const auto & gcSettings = config->getLocalSettings().getGCSettings(); bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool keepOutputs = gcSettings.keepOutputs; - bool keepDerivations = gcSettings.keepDerivations; boost::unordered_flat_set> roots, dead, alive; /* Return early if nothing to delete */ - if (std::holds_alternative(options.pathsToDelete) - && std::get(options.pathsToDelete).empty()) + if (std::visit( + overloaded{ + [](const GCOptions::SpecificPaths & pathsToDelete) { return pathsToDelete.paths.empty(); }, + [](const GCOptions::WholeStore & _) { return false; }}, + options.pathsToDelete)) return; struct Shared @@ -382,15 +385,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::condition_variable wakeup; - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - keepOutputs = false; - keepDerivations = false; - } - if (shouldDelete) deletePath(reservedPath); @@ -601,6 +595,33 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) todo.push(path); }; + auto markAlive = [&](const StorePath & p) { + alive.insert(p); + try { + StorePathSet closure; + bool includeOutputs = false; + bool includeDerivers = false; + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + includeOutputs = gcSettings.keepOutputs; + includeDerivers = gcSettings.keepDerivations; + }, + [](const GCOptions::SpecificPaths &) {}, + }, + options.pathsToDelete); + computeFSClosure( + p, + closure, + /* flipDirection */ false, + includeOutputs, + includeDerivers); + for (auto & c : closure) + alive.insert(c); + } catch (InvalidPath &) { + } + }; + enqueue(start); while (auto path = pop(todo)) { @@ -608,49 +629,48 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Bail out if we've previously discovered that this path is alive. */ - if (alive.count(*path)) { + if (alive.contains(*path)) { + debug("cannot delete '%s' because '%s' is alive", printStorePath(start), printStorePath(*path)); alive.insert(start); return; } /* If we've previously deleted this path, we don't have to handle it again. */ - if (dead.count(*path)) + if (dead.contains(*path)) continue; - auto markAlive = [&]() { - alive.insert(*path); - alive.insert(start); - try { - StorePathSet closure; - computeFSClosure( - *path, - closure, - /* flipDirection */ false, - keepOutputs, - keepDerivations); - for (auto & p : closure) - alive.insert(p); - } catch (InvalidPath &) { - } - }; - /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (roots.contains(*path)) { debug("cannot delete '%s' because it's a root", printStorePath(*path)); - return markAlive(); + alive.insert(start); + return markAlive(*path); } - if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) + if (std::visit( + overloaded{ + [&](const GCOptions::SpecificPaths & pathsToDelete) { + if (!pathsToDelete.deleteReferrers && !pathsToDelete.paths.contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); + return true; + } + return false; + }, + [](const GCOptions::WholeStore & _) { return false; }, + }, + options.pathsToDelete)) return; { auto hashPart = path->hashPart(); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { + if (shared->tempRoots.contains(hashPart)) { debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); - return markAlive(); + alive.insert(start); + return markAlive(*path); } shared->pending = hashPart; } @@ -668,27 +688,60 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a - derivation, then visit the derivation outputs. */ - if (keepDerivations && path->isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) - if (maybeOutPath && isValidPath(*maybeOutPath) - && queryPathInfo(*maybeOutPath)->deriver == *path) - enqueue(*maybeOutPath); - } - - /* If keep-outputs is set, then visit the derivers. */ - if (keepOutputs) { - auto derivers = queryValidDerivers(*path); - for (auto & i : derivers) - enqueue(i); - } + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + /* If keep-derivations is set and this is a derivation, then we only want to delete this + * derivation if we can also delete all its outputs, so visit the derivation outputs. */ + if (gcSettings.keepDerivations && path->isDerivation()) + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && isValidPath(*maybeOutPath) + && queryPathInfo(*maybeOutPath)->deriver == path) + enqueue(*maybeOutPath); + + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ + if (gcSettings.keepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + }, + [](const GCOptions::SpecificPaths &) {}, + }, + options.pathsToDelete); } } for (auto & path : topoSortPaths(visited)) { if (!dead.insert(path).second) continue; if (shouldDelete) { + /* Re-check tempRoots before deleting and set pending + to synchronise with addTempRoot. Between the BFS + and this deletion loop, new temproots may have been + added via the GC socket by a concurrent process + (e.g. an evaluator calling addTempRoot). The BFS + only checks tempRoots when it first visits a path, + but the "pending" mechanism only blocks the socket + handler for the single path currently being visited, + not for paths already queued for deletion. */ + { + auto hashPart = std::string(path.hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.contains(hashPart)) { + debug( + "not deleting '%s' because it became a temporary root after initial scan", + printStorePath(path)); + markAlive(path); + continue; + } + shared->pending = hashPart; + } + Finally resetPending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); try { invalidatePathChecked(path); deleteFromStore(path.to_string(), true); @@ -706,28 +759,30 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Either delete all garbage paths, or just the specified paths. */ std::visit( overloaded{ - [&](const StorePathSet & paths) { - for (auto & i : paths) { - switch (options.action) { - case GCOptions::gcDeleteDead: - printInfo("deleting garbage within specified paths..."); - break; - case GCOptions::gcDeleteSpecific: - printInfo("deleting specified paths..."); - break; - case GCOptions::gcReturnDead: - case GCOptions::gcReturnLive: - printInfo("determining live/dead paths..."); - } + [&](const GCOptions::SpecificPaths & pathsToDelete) { + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } + for (auto & i : pathsToDelete.paths) { maybeDeleteReferrersClosure(i); - if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) + if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " "nix-store --query --roots and nix-store --query --referrers", printStorePath(i)); + else if (!dead.contains(i)) + debug("cannot delete '%s' because it's still alive", printStorePath(i)); } }, [&](const GCOptions::WholeStore & _) { @@ -883,12 +938,17 @@ void LocalStore::autoGC(bool sync) if (avail > state->availAfterGC * 0.97) return; + /* Note: since gcRunning is false here, any previous GC thread has exited / is exiting so the join() should be + * almost instantenous. */ + if (state->gcThread.joinable()) + state->gcThread.join(); + state->gcRunning = true; std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { + state->gcThread = std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -915,7 +975,7 @@ void LocalStore::autoGC(bool sync) // future, but we don't really care. (what??) ignoreExceptionInDestructor(); } - }).detach(); + }); } sync: diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8beb68aa8e33..0bbd4b9f09cd 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,6 +4,7 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" +#include "nix/util/file-system.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" @@ -44,6 +45,14 @@ namespace nix { +void Settings::anchor() {} + +void NarInfoDiskCacheSettings::anchor() {} + +void LogFileSettings::anchor() {} + +void AutoAllocateUidSettings::anchor() {} + Settings settings; static GlobalConfig::Register rSettings(&settings); @@ -246,11 +255,11 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. + // + // The runtime file exists iff Rosetta 2 is installed; checking it avoids + // spawning a subprocess during static initialization of `settings`. if (std::string{NIX_LOCAL_SYSTEM} == "aarch64-darwin" - && runProgram( - RunOptions{.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}) - .first - == 0) + && pathExists("/Library/Apple/usr/libexec/oah/libRosettaRuntime")) extraPlatforms.insert("x86_64-darwin"); #endif @@ -270,7 +279,7 @@ bool Settings::isWSL1() #endif } -const ExternalBuilder * LocalSettings::findExternalDerivationBuilderIfSupported(const Derivation & drv) +const ExternalBuilder * LocalSettings::findExternalDerivationBuilderIfSupported(const BasicDerivation & drv) { if (auto it = std::ranges::find_if( externalBuilders.get(), [&](const auto & handler) { return handler.systems.contains(drv.platform); }); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ff5a89f135bb..d881e696b756 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -14,6 +14,8 @@ namespace nix { MakeError(UploadToHTTP, Error); +void UploadToHTTP::anchor() {} + StringSet HttpBinaryCacheStoreConfig::uriSchemes() { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; @@ -23,6 +25,10 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } +void HttpBinaryCacheStoreConfig::anchor() {} + +void HttpBinaryCacheStore::anchor() {} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) : StoreConfig(params, FilePathType::Unix) , BinaryCacheStoreConfig(params) @@ -78,7 +84,8 @@ void HttpBinaryCacheStore::init() } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } - diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority); + diskCache->createCache( + cacheKey, config->storeDir, {.wantMassQuery = config->wantMassQuery, .priority = config->priority}); } } @@ -214,7 +221,7 @@ void HttpBinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToHTTP err(e.message()); err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md index 20c26d0c2caf..03dd350ec518 100644 --- a/src/libstore/http-binary-cache-store.md +++ b/src/libstore/http-binary-cache-store.md @@ -2,7 +2,7 @@ R"( **Store URL format**: `http://...`, `https://...` -This store allows a binary cache to be accessed via the HTTP +This store allows a [binary cache](@docroot@/protocols/binary-cache/index.md) to be accessed via the HTTP protocol. )" diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index 0751757cb015..3fa747069935 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -36,6 +36,10 @@ struct AwsCredentials class AwsAuthError final : public CloneableError { +private: + void anchor() override; + +public: std::optional errorCode; public: @@ -70,7 +74,7 @@ public: } } - virtual ~AwsCredentialProvider() {} + virtual ~AwsCredentialProvider(); }; /** diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 7871ad03c884..0cc5d1f3ff3f 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -16,6 +16,10 @@ class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: BinaryCacheStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { @@ -92,6 +96,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; private: + void anchor() override; + std::vector> signers; protected: @@ -177,6 +183,23 @@ private: void writeNarInfo(ref narInfo); + /** + * Upload the NAR for a path and everything else *except* the + * `.narinfo` file (i.e. the compressed NAR, an optional NAR + * listing, and optional debuginfo links), and construct the + * corresponding `NarInfo`. The returned `NarInfo` is neither signed + * nor published yet; call `uploadNarInfo()` to do that. + */ + ref uploadData(Source & narSource, RepairFlag repair, fun mkInfo); + + /** + * Sign and publish the `.narinfo` file for a path whose NAR has + * already been uploaded by `uploadData()`. This is what establishes + * the closure invariant, so all of the path's references must + * already be valid in the store. + */ + void uploadNarInfo(ref narInfo); + ref addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo); @@ -197,6 +220,9 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; + void + addMultipleToStore(PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) override; + StorePath addToStoreFromDump( Source & dump, std::string_view name, diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index c664e6e5b6f7..808fcc829eda 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -58,8 +58,11 @@ enum struct BuildResultFailureStatus : uint8_t { * This is both an exception type (inherits from Error) and serves as * the failure variant in BuildResult::inner. */ -struct BuildError : public CloneableError +class BuildError : public CloneableError { + void anchor() override; + +public: using Status = BuildResultFailureStatus; using enum Status; diff --git a/src/libstore/include/nix/store/build.hh b/src/libstore/include/nix/store/build.hh new file mode 100644 index 000000000000..2c7bb1211804 --- /dev/null +++ b/src/libstore/include/nix/store/build.hh @@ -0,0 +1,129 @@ +#pragma once +///@file + +#include "nix/store/store-api.hh" + +namespace nix { + +/** + * Abstract interface for the build scheduler entry points. + * + * `Worker` implements this for local scheduling, including local builds. + * Remote stores provide a `Builder` via `Store::getBuilder()`. + * + * Thread safety should be guaranteed across these methods. + */ +struct Builder +{ + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ + virtual void buildPaths(const std::vector & reqs, BuildMode buildMode = bmNormal) = 0; + + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ + virtual std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode = bmNormal) = 0; + + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ + virtual BuildResult + buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; + + /** + * Like the other buildDerivation(), but additionally copies a set of + * input paths into the builder's store before the build is run. + * + * This lets the caller ship the build inputs together with the build + * request, rather than as a separate prior `copyPaths()`. Whether the + * inputs are fetched via substitution or copied directly is governed by + * the `builders-use-substitutes` setting. + * + * @param inputs The store paths to make available in the builder's + * store before building. For a remote builder these are copied across + * the connection; for the local `Worker` they are copied from the eval + * store. + */ + virtual BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode = bmNormal) = 0; + + /** + * Like the other buildPathsWithResults(), but additionally copies a set + * of input paths into the builder's store before building. + * + * @param inputs The store paths to make available in the builder's + * store before building, copied subject to the + * `builders-use-substitutes` setting (see the buildDerivation() overload + * above). + */ + virtual std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode = bmNormal) = 0; + + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ + virtual void ensurePath(const StorePath & path) = 0; + + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ + virtual void repairPath(const StorePath & path) = 0; + + virtual ~Builder() = default; +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/build/build-log.hh b/src/libstore/include/nix/store/build/build-log.hh index cdc9125734d1..5dbdf2ba8680 100644 --- a/src/libstore/include/nix/store/build/build-log.hh +++ b/src/libstore/include/nix/store/build/build-log.hh @@ -31,6 +31,8 @@ private: std::string currentLogLine; size_t currentLogLinePos = 0; // to handle carriage return + bool pendingCR = false; // defer '\r' so "\r\n" is treated as a line terminator + void flushLine(); public: diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index bff249f78154..088644a3eb32 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -5,6 +5,7 @@ #include #include "nix/store/build-result.hh" +#include "nix/store/daemon.hh" #include "nix/store/derivation-options.hh" #include "nix/store/build/derivation-building-misc.hh" #include "nix/store/derivations.hh" @@ -16,12 +17,27 @@ namespace nix { +/** + * Rethrow the current exception as a subclass of `Error`. + */ +void rethrowExceptionAsError(); + +/** + * Send the current exception to the parent in the format expected by + * `DerivationBuilderImpl::processSandboxSetupMessages()`. + */ +void handleChildException(bool sendException); + /** * Denotes a build failure that stemmed from the builder exiting with a * failing exist status. */ struct BuilderFailureError final : CloneableError { +private: + void anchor() override; + +public: int builderStatus; std::string extraMsgAfter; @@ -109,7 +125,7 @@ struct DerivationBuilderParams */ struct DerivationBuilderCallbacks { - virtual ~DerivationBuilderCallbacks() = default; + virtual ~DerivationBuilderCallbacks(); /** * Open a log file and a pipe to it. @@ -125,6 +141,17 @@ struct DerivationBuilderCallbacks * @todo this should be reworked */ virtual void childTerminated() = 0; + + /** + * Process a recursive Nix daemon connection, using a builder + * that enforces the restrictions of the given context. + */ + virtual void processDaemonConnection( + ref store, + FdSource && from, + FdSink && to, + RestrictionContext & context, + daemon::RecursiveFlag recursiveFlag) = 0; }; /** @@ -140,6 +167,10 @@ struct DerivationBuilderCallbacks */ struct DerivationBuilder : RestrictionContext { +private: + void anchor() override; + +public: DerivationBuilder() = default; virtual ~DerivationBuilder() = default; @@ -215,7 +246,7 @@ using DerivationBuilderUnique = std::unique_ptr miscMethods, DerivationBuilderParams params); + LocalStore & store, std::shared_ptr miscMethods, DerivationBuilderParams params); /** * @param handler Must be chosen such that it supports the given @@ -223,7 +254,7 @@ DerivationBuilderUnique makeDerivationBuilder( */ DerivationBuilderUnique makeExternalDerivationBuilder( LocalStore & store, - std::unique_ptr miscMethods, + std::shared_ptr miscMethods, DerivationBuilderParams params, const ExternalBuilder & handler); #endif diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 6a17f73eeb51..08b3fa80d4b2 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -34,20 +34,11 @@ struct DerivationBuildingGoal : public Goal friend class Worker; /** - * @param storeDerivation Whether to store the derivation in - * `worker.store`. This is useful for newly-resolved derivations. In this - * case, the derivation was not created a priori, e.g. purely (or close - * enough) from evaluation of the Nix language, but also depends on the - * exact content produced by upstream builds. It is strongly advised to - * have a permanent record of such a resolved derivation in order to - * faithfully reconstruct the build history. + * @param drv The derivation to build, with the outputs of its input + * derivations already added to its input sources. */ DerivationBuildingGoal( - const StorePath & drvPath, - ref drv, - Worker & worker, - BuildMode buildMode, - bool storeDerivation); + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); ~DerivationBuildingGoal(); private: @@ -56,9 +47,9 @@ private: const StorePath drvPath; /** - * The derivation stored at drvPath. + * The derivation to build. */ - const ref drv; + const ref drv; /** * The remainder is state held during the build. @@ -79,7 +70,7 @@ private: /** * The states. */ - Co gaveUpOnSubstitution(bool storeDerivation); + Co gaveUpOnSubstitution(); Co tryToBuild(StorePathSet inputPaths); Co buildWithHook( StorePathSet inputPaths, diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 8d6892839c76..dfdc25d0ed1a 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -9,7 +9,11 @@ namespace nix { class Store; -struct Derivation; + +template +struct DerivationT; +struct FullInputs; +using Derivation = DerivationT; /** * Unless we are repairing, we don't both to test validity and just assume it, @@ -51,6 +55,7 @@ struct InitialOutput /** * Format the known outputs of a derivation for use in error messages. */ -std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv); +template +std::string showKnownOutputs(const StoreDirConfig & store, const DerivationT & drv); } // namespace nix diff --git a/src/libstore/include/nix/store/build/derivation-env-desugar.hh b/src/libstore/include/nix/store/build/derivation-env-desugar.hh index a10ec9fa8736..ce65c7cbe8cf 100644 --- a/src/libstore/include/nix/store/build/derivation-env-desugar.hh +++ b/src/libstore/include/nix/store/build/derivation-env-desugar.hh @@ -7,7 +7,13 @@ namespace nix { class Store; -struct Derivation; + +template +struct DerivationT; +struct FullInputs; +using Derivation = DerivationT; +using BasicDerivation = DerivationT; + template struct DerivationOptions; @@ -79,7 +85,7 @@ struct DesugaredEnv */ static DesugaredEnv create( Store & store, - const Derivation & drv, + const BasicDerivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths); }; diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index 843e4031aa7e..972558e9706f 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -37,7 +37,8 @@ struct DerivationResolutionGoal : public Goal { friend class Worker; - DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); + DerivationResolutionGoal( + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); /** * If the derivation needed to be resolved, this is resulting @@ -55,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 0b7367ff7e02..f0861ddb4794 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -10,8 +10,11 @@ namespace nix { -struct TimedOut final : CloneableError +class TimedOut final : public CloneableError { + void anchor() override; + +public: time_t maxDuration; TimedOut(time_t maxDuration); @@ -75,6 +78,47 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); +public: + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant>; + +private: + class ChildEvents + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::unique_ptr childTimeout; + + public: + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + }; + /** * Goals that this goal is waiting for. */ @@ -85,6 +129,8 @@ private: */ std::optional cachedKey; + ChildEvents childEvents; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -152,22 +198,6 @@ public: friend Goal; }; - /** - * Event types for child process communication, delivered via coroutines. - */ - struct ChildOutput - { - Descriptor fd; - std::string data; - }; - - struct ChildEOF - { - Descriptor fd; - }; - - using ChildEvent = std::variant; - /** * Tag type for `co_await`-ing child events. * Returns a `ChildEvent` when resumed. @@ -233,8 +263,10 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - void operator=(Co &&); - Co(Co && rhs); + Co & operator=(Co &&) noexcept; + Co(Co && rhs) noexcept; + Co & operator=(const Co &) = delete; + Co(const Co & rhs) = delete; ~Co(); bool await_ready() @@ -319,28 +351,6 @@ public: */ bool alive = true; - class - { - /** - * Structured queue of child events: - * - outputs: stream of data from child - * - eof: optional end-of-stream marker - * - timeout: optional timeout that flushes/overrides other events - */ - std::queue childOutputs; - std::optional childEOF; - std::optional childTimeout; - - public: - - void pushChildEvent(ChildOutput event); - void pushChildEvent(ChildEOF event); - void pushChildEvent(TimedOut event); - bool hasChildEvent() const; - ChildEvent popChildEvent(); - - } childEvents; - /** * The awaiter used by @ref final_suspend. */ @@ -448,7 +458,7 @@ public: bool await_ready() { - assert(!promise.childEvents.hasChildEvent()); + assert(!promise.goal->childEvents.hasChildEvent()); return false; } @@ -476,7 +486,7 @@ public: bool await_ready() { - return handle && handle.promise().childEvents.hasChildEvent(); + return handle && handle.promise().goal->childEvents.hasChildEvent(); } void await_suspend(handle_type h) @@ -487,7 +497,7 @@ public: ChildEvent await_resume() { assert(handle); - return handle.promise().childEvents.popChildEvent(); + return handle.promise().goal->childEvents.popChildEvent(); } }; diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 4c836986811f..bf1726f32b90 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -3,6 +3,7 @@ #include "nix/util/types.hh" #include "nix/store/store-api.hh" +#include "nix/store/build.hh" #include "nix/store/derived-path-map.hh" #include "nix/store/build/goal.hh" #include "nix/store/build-result.hh" @@ -67,10 +68,51 @@ struct Child struct HookInstance; #endif +/** + * Owns a worker. Optimization around ensurePath to prevent a Worker from + * being constructed when it's not needed. + */ +class LocalBuilder : public Builder +{ +public: + LocalBuilder(ref store, ref evalStore) + : store(store) + , evalStore(evalStore) {}; + + /* Builder interface — see `Builder` for documentation. */ + + void buildPaths(const std::vector & reqs, BuildMode buildMode) override; + std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override; + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override; + void ensurePath(const StorePath & path) override; + void repairPath(const StorePath & path) override; + +private: + /** + * Intentionally construct a new worker for each operation, to avoid + * reusing a worker between calls, allowing for thread safety. + */ + inline std::shared_ptr getWorker() + { + return std::make_shared(*store, *evalStore); + } + + ref store; + ref evalStore; +}; + /** * Coordinates one or more realisations and their interdependencies. */ -class Worker +class Worker : public Builder { private: @@ -199,6 +241,7 @@ public: Store & store; Store & evalStore; + const WorkerSettings & settings; /** @@ -265,13 +308,13 @@ public: * @ref DerivationResolutionGoal "derivation resolution goal" */ std::shared_ptr - makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode); + makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref DerivationBuildingGoal "derivation building goal" */ - std::shared_ptr makeDerivationBuildingGoal( - const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation); + std::shared_ptr + makeDerivationBuildingGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref PathSubstitutionGoal "substitution goal" @@ -388,6 +431,22 @@ public: act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } + + /* Builder interface — see `Builder` for documentation. */ + + void buildPaths(const std::vector & reqs, BuildMode buildMode) override; + std::vector + buildPathsWithResults(const std::vector & reqs, BuildMode buildMode) override; + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + BuildResult buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + const StorePathSet & inputs, + BuildMode buildMode) override; + std::vector buildPathsWithResults( + const std::vector & reqs, const StorePathSet & inputs, BuildMode buildMode) override; + void ensurePath(const StorePath & path) override; + void repairPath(const StorePath & path) override; }; } // namespace nix diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0f0b3f24b99..b528871b2801 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -25,6 +25,9 @@ struct Package class BuildEnvFileConflictError final : public CloneableError { +private: + void anchor() override; + public: const std::filesystem::path fileA; const std::filesystem::path fileB; diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 1e90c94afcbe..b622e82e26bf 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -10,6 +10,10 @@ class SSHMaster; struct CommonSSHStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: CommonSSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/content-address.hh b/src/libstore/include/nix/store/content-address.hh index 41ccc69aeb3f..ce700bd1ac0d 100644 --- a/src/libstore/include/nix/store/content-address.hh +++ b/src/libstore/include/nix/store/content-address.hh @@ -131,6 +131,15 @@ struct ContentAddressMethod * for hashing file systeme objects. */ FileIngestionMethod getFileIngestionMethod() const; + + /** + * The FileSerialisationMethod that is recommended for this content addressing method. + * In some circumstances, other methods are also valid. + * Note that `Git` is mapped to `NixArchive`, even though it could represent a single file. + * There is no support for flat serialisation of merkle objects; and even if there were, + * it would be unable to represent executable files. + */ + FileSerialisationMethod getFileSerialisationMethod() const; }; /* diff --git a/src/libstore/include/nix/store/daemon.hh b/src/libstore/include/nix/store/daemon.hh index 4d550696e877..a0a0d63e30b0 100644 --- a/src/libstore/include/nix/store/daemon.hh +++ b/src/libstore/include/nix/store/daemon.hh @@ -4,10 +4,26 @@ #include "nix/util/serialise.hh" #include "nix/store/store-api.hh" -namespace nix::daemon { +namespace nix { -enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; +struct Builder; -void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); +namespace daemon { -} // namespace nix::daemon +enum struct RecursiveFlag { + NotRecursive = 0, + Recursive = 1, + RecursiveSubmitted = 2, +}; + +void processConnection( + ref store, + FdSource && from, + FdSink && to, + TrustedFlag trusted, + RecursiveFlag recursive, + std::shared_ptr builder = nullptr); + +} // namespace daemon + +} // namespace nix diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index e29f660c4848..0931b41143f7 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -13,8 +13,15 @@ namespace nix { +class Store; + struct StoreDirConfig; -struct BasicDerivation; + +template +struct DerivationT; +struct FullInputs; +using BasicDerivation = DerivationT; + struct StructuredAttrs; template @@ -180,14 +187,16 @@ struct DerivationOptions * the future we'll flip things around so a `BasicDerivation` has * `DerivationOptions` instead. */ - StringSet getRequiredSystemFeatures(const BasicDerivation & drv) const; + template + StringSet getRequiredSystemFeatures(const DerivationT & drv) const; bool substitutesAllowed(const WorkerSettings & workerSettings) const; /** * @param drv See note on `getRequiredSystemFeatures` */ - bool useUidRange(const BasicDerivation & drv) const; + template + bool useUidRange(const DerivationT & drv) const; }; extern template struct DerivationOptions; diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 4cfc79acbe15..bfb20846ac34 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -16,6 +16,11 @@ namespace nix { +/** + * String to include in requiredSystemFeatures to enable builder-rpc-v0 + */ +static constexpr std::string_view drvFeatureBuilderRpcV0 = "builder-rpc-v0"; + struct StoreDirConfig; /* Abstract syntax of derivations. */ @@ -152,6 +157,23 @@ typedef std::map DerivationInputs; +/** + * Inputs for full Derivation - both source and derivation inputs + */ +struct FullInputs +{ + /** + * inputs that are sources + */ + StorePathSet srcs; + /** + * inputs that are sub-derivations + */ + DerivedPathMap>> drvs; + + bool operator==(const FullInputs &) const = default; +}; + struct DerivationType { /** @@ -263,16 +285,20 @@ struct DerivationType bool hasKnownOutputPaths() const; }; -struct BasicDerivation +template +struct DerivationT; + +using BasicDerivation = DerivationT; +using Derivation = DerivationT; + +template +struct DerivationT { /** * keyed on symbolic IDs */ DerivationOutputs outputs; - /** - * inputs that are sources - */ - StorePathSet inputSrcs; + Inputs inputs; std::string platform; /** * Probably should be an absolute path in the path format that `platform` uses @@ -287,12 +313,7 @@ struct BasicDerivation std::string name; - BasicDerivation() = default; - BasicDerivation(BasicDerivation &&) = default; - BasicDerivation(const BasicDerivation &) = default; - BasicDerivation & operator=(BasicDerivation &&) = default; - BasicDerivation & operator=(const BasicDerivation &) = default; - virtual ~BasicDerivation() {}; + bool operator==(const DerivationT &) const = default; bool isBuiltin() const; @@ -321,27 +342,14 @@ struct BasicDerivation */ void applyRewrites(const StringMap & rewrites); - bool operator==(const BasicDerivation &) const = default; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - // auto operator <=> (const BasicDerivation &) const = default; -}; - -class Store; - -struct Derivation : BasicDerivation -{ - /** - * inputs that are sub-derivations - */ - DerivedPathMap>> inputDrvs; - /** - * Print a derivation. + * Print a derivation (only meaningful for full Derivation). */ std::string unparse( const StoreDirConfig & store, bool maskOutputs, - DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; + DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const + requires std::is_same_v; /** * Determine whether this derivation should be resolved before building. @@ -354,7 +362,8 @@ struct Derivation : BasicDerivation * - Impure derivations always need resolution * - Any input derivations have outputs from dynamic derivations */ - bool shouldResolve() const; + bool shouldResolve() const + requires std::is_same_v; /** * Return the underlying basic derivation but with these changes: @@ -365,7 +374,8 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const + requires std::is_same_v; /** * Like the above, but instead of querying the Nix database for @@ -375,7 +385,34 @@ struct Derivation : BasicDerivation std::optional tryResolve( Store & store, fun(ref drvPath, const std::string & outputName)> - queryResolutionChain) const; + queryResolutionChain) const + requires std::is_same_v; + + /** + * Convert a BasicDerivation to a full Derivation. + * The resulting Derivation has empty inputDrvs since BasicDerivation + * is already resolved. + */ + Derivation unresolve() const + requires std::is_same_v; + + /** + * Return a derivation identical to this one, but with the inputs transformed by `f`. + */ + template + DerivationT> mapInputs(F f) const + { + return { + .outputs = outputs, + .inputs = f(inputs), + .platform = platform, + .builder = builder, + .args = args, + .env = env, + .structuredAttrs = structuredAttrs, + .name = name, + }; + } /** * Check that the derivation is valid and does not present any @@ -424,19 +461,8 @@ struct Derivation : BasicDerivation * @param store The store to use for path computation * @param drvName The derivation name (without .drv extension) */ - void fillInOutputPaths(Store & store); - - Derivation() = default; - - Derivation(const BasicDerivation & bd) - : BasicDerivation(bd) - { - } - - Derivation(BasicDerivation && bd) - : BasicDerivation(std::move(bd)) - { - } + void fillInOutputPaths(Store & store) + requires std::is_same_v; /** * Parse a derivation from JSON, and also perform various @@ -459,15 +485,35 @@ struct Derivation : BasicDerivation * @return A validated derivation with output paths filled in * @throws Error if parsing fails, output paths can't be computed, or validation fails */ - static Derivation parseJsonAndValidate(Store & store, const nlohmann::json & json); - - bool operator==(const Derivation &) const = default; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - // auto operator <=> (const Derivation &) const = default; + static Derivation parseJsonAndValidate(Store & store, const nlohmann::json & json) + requires std::is_same_v; }; class Store; +template<> +std::string DerivationT::unparse( + const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const; +template<> +bool DerivationT::shouldResolve() const; +template<> +std::optional DerivationT::tryResolve(Store & store, Store * evalStore) const; +template<> +std::optional DerivationT::tryResolve( + Store & store, + fun(ref drvPath, const std::string & outputName)> + queryResolutionChain) const; +template<> +void DerivationT::fillInOutputPaths(Store & store); +template<> +Derivation DerivationT::parseJsonAndValidate(Store & store, const nlohmann::json & json); +template<> +Derivation DerivationT::unresolve() const; +template<> +void DerivationT::checkInvariants(Store & store) const; +template<> +void DerivationT::checkInvariants(Store & store) const; + /** * Compute the store path that would be used for a derivation without writing it. * @@ -620,5 +666,8 @@ constexpr unsigned expectedJsonVersionDerivation = 4; } // namespace nix JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) -JSON_IMPL_WITH_XP_FEATURES(nix::BasicDerivation) -JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) + +namespace nlohmann { +template +JSON_IMPL_WITH_XP_FEATURES_INNER(nix::DerivationT); +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/downstream-placeholder.hh b/src/libstore/include/nix/store/downstream-placeholder.hh index ba3e9faeff70..6fe252bf938d 100644 --- a/src/libstore/include/nix/store/downstream-placeholder.hh +++ b/src/libstore/include/nix/store/downstream-placeholder.hh @@ -37,7 +37,7 @@ using DrvRef = std::variant; * We use them with `Derivation`: the `render()` method is called to * render an opaque string which can be used in the derivation, and the * resolving logic can substitute those strings for store paths when - * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`. + * resolving `Derivation.inputs.drvs` to `BasicDerivation.inputs.srcs`. */ class DownstreamPlaceholder { diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 8fdeeb362515..bec77a6bee68 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -15,6 +15,10 @@ struct MemorySourceAccessor; */ struct DummyStore : virtual Store { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index f76fb3d5c2b0..c8a212c75603 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -12,6 +12,10 @@ struct DummyStore; struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { +private: + void anchor() override; + +public: DummyStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index a423249e6634..774a4e1403b3 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -23,16 +23,29 @@ namespace nix { const std::filesystem::path & nixConfDir(); -struct FileTransferSettings : Config +class FileTransferSettings : public Config { -private: static std::optional getDefaultSSLCertFile(); + void anchor() override; + public: FileTransferSettings(); Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; + Setting enableHttp3{ + this, + false, + "http3", + R"( + Whether to try enabling HTTP/3 (QUIC). + When enabled, Nix requests HTTP/3 and transparently falls back + to HTTP/2 or HTTP/1.1 for servers that do not support it. + This option has no effect unless the `nix` binary is linked + against a libcurl built with HTTP/3 (QUIC) support. + )"}; + Setting userAgentSuffix{ this, "", "user-agent-suffix", "String appended to the user agent in HTTP requests."}; @@ -307,6 +320,14 @@ struct FileTransferRequest { } + /** + * `uri` with any userinfo (`user:password@`) stripped, for use in + * progress, warning and error messages so credentials embedded in + * the URL don't leak into logs. Returns `uri` verbatim if it can't + * be parsed. + */ + std::string displayUri() const; + /** * Returns the method description for logging purposes. */ @@ -410,7 +431,7 @@ public: } }; - virtual ~FileTransfer() {} + virtual ~FileTransfer(); /** * Enqueue a data transfer request, returning a future to the result of @@ -449,7 +470,7 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; /** @@ -469,6 +490,9 @@ ref makeFileTransfer(const FileTransferSettings & settings = fileT class FileTransferError final : public CloneableError { +private: + void anchor() override; + public: FileTransfer::Error error; /// intentionally optional diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 5e23f2052472..d7637374157f 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -42,6 +42,16 @@ struct GCOptions struct WholeStore {}; + struct SpecificPaths + { + StorePathSet paths; + + /** + * Allow dead referrers of candidate paths to also be deleted. + */ + bool deleteReferrers = false; + }; + GCAction action{gcDeleteDead}; /** @@ -55,7 +65,7 @@ struct GCOptions /** * The paths from which to delete. */ - using GCPaths = std::variant; + using GCPaths = std::variant; GCPaths pathsToDelete; /** @@ -106,6 +116,10 @@ struct GCResults */ struct GcStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Garbage collection"; /** @@ -121,6 +135,13 @@ struct GcStore : public virtual Store * Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /** + * Delete build trace entries (realisations) from the store's database. + * + * The entries are specified by their key (the build trace is a map). + */ + virtual void deleteBuildTraces(const std::set & keys) = 0; }; } // namespace nix diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index becab079198f..34eab7f88469 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -19,6 +19,10 @@ struct ProfileDirsOptions; struct LogFileSettings : public virtual Config { +private: + void anchor() override; + +public: Setting keepLog{ this, true, @@ -45,6 +49,10 @@ struct LogFileSettings : public virtual Config struct NarInfoDiskCacheSettings : public virtual Config { +private: + void anchor() override; + +public: Setting ttlNegative{ this, 3600, @@ -96,6 +104,9 @@ class Settings : public virtual Config, private WorkerSettings, private NarInfoDiskCacheSettings { +private: + void anchor() override; +public: StringSet getDefaultSystemFeatures(); StringSet getDefaultExtraPlatforms(); diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 748daec646b9..12465261caef 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -14,6 +14,10 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: LegacySSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , CommonSSHStoreConfig(params) @@ -63,6 +66,10 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this struct LegacySSHStore : public virtual Store { +private: + void anchor() override; + +public: using Config = LegacySSHStoreConfig; ref config; @@ -132,24 +139,7 @@ struct LegacySSHStore : public virtual Store public: - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - - /** - * Note, the returned function must only be called once, or we'll - * try to read from the connection twice. - * - * @todo Use C++23 `std::move_only_function`. - */ - fun buildDerivationAsync( - const StorePath & drvPath, const BasicDerivation & drv, const ServeProto::BuildOptions & options); - - void buildPaths( - const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; - - void ensurePath(const StorePath & path) override - { - unsupported("ensurePath"); - } + ref getBuilder(std::shared_ptr evalStore) override; ref getFSAccessor(bool requireValidPath) override { @@ -161,19 +151,6 @@ public: unsupported("getFSAccessor"); } - /** - * The default instance would schedule the work on the client side, but - * for consistency with `buildPaths` and `buildDerivation` it should happen - * on the remote side. - * - * We make this fail for now so we can add implement this properly later - * without it being a breaking change. - */ - void repairPath(const StorePath & path) override - { - unsupported("repairPath"); - } - void computeFSClosure( const StorePathSet & paths, StorePathSet & out, @@ -224,6 +201,8 @@ public: // not supported return {}; } + + friend struct LegacySSHBuilder; }; } // namespace nix diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 69a4bac1c8a9..181b33e4bdf8 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -9,6 +9,10 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this> makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { @@ -45,7 +47,7 @@ public: R"( Directory where Nix stores state. - Defaults to [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) when [`root`](#store-setting-root) is not set. + Defaults to [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) when [`root`](#@store-slug@-root) is not set. )", }; @@ -56,7 +58,7 @@ public: R"( Directory where Nix stores log files. - Defaults to [`NIX_LOG_DIR`](@docroot@/command-ref/env-common.md#env-NIX_LOG_DIR) when [`root`](#store-setting-root) is not set. + Defaults to [`NIX_LOG_DIR`](@docroot@/command-ref/env-common.md#env-NIX_LOG_DIR) when [`root`](#@store-slug@-root) is not set. )", }; @@ -67,7 +69,7 @@ public: R"( Physical path of the Nix store. - Defaults to [`store`](#store-setting-store) when [`root`](#store-setting-root) is not set. + Defaults to [`store`](#@store-slug@-store) when [`root`](#@store-slug@-root) is not set. )", }; @@ -87,6 +89,10 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ virtual GcStore, virtual LogStore { +private: + void anchor() override; + +public: using Config = LocalFSStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index dfb1fb184a55..10b04937c3aa 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -7,6 +7,10 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { +private: + void anchor() override; + +public: LocalOverlayStoreConfig(const StringMap & params) : LocalOverlayStoreConfig("", params) { @@ -119,6 +123,8 @@ struct LocalOverlayStore : virtual LocalStore LocalOverlayStore(ref); private: + void anchor() override; + /** * The store beneath us. * diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 4fe28818d2b8..46186fd6ae1b 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -39,6 +39,10 @@ void BaseSetting::appendOrSet(PathsInChroot newValue, bool append struct GCSettings : public virtual Config { +private: + void anchor() override; + +public: Setting reservedSize{ this, 8 * 1024 * 1024, @@ -61,6 +65,9 @@ struct GCSettings : public virtual Config collector still deletes store paths that are used only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to `true`. + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-outputs"}, }; @@ -80,6 +87,9 @@ struct GCSettings : public virtual Config store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if `keep-outputs` is also turned on). + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-derivations"}, }; @@ -128,6 +138,10 @@ const uint32_t maxIdsPerBuild = struct AutoAllocateUidSettings : public virtual Config { +private: + void anchor() override; + +public: Setting startId{ this, #ifdef __linux__ @@ -163,6 +177,10 @@ struct AutoAllocateUidSettings : public virtual Config */ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllocateUidSettings { +private: + void anchor() override; + +public: /** * Get the GC settings. */ @@ -190,7 +208,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#env-vars) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism.