From 980d0599ac44887341861ceef0138e2738d2e9ce Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 3 Aug 2026 02:07:07 +0200 Subject: [PATCH 1/4] feat: add camoufox support default for potoken generation --- .github/workflows/deno-check.yaml | 29 +- .github/workflows/release-binaries.yaml | 9 +- Dockerfile | 98 ++++++- README.md | 10 +- deno.jsonc | 12 +- deno.lock | 330 ++++++++++++++++++++- dependencies.json | 14 + docker-compose.yaml | 1 + docker/camoufox-bootstrap.go | 210 +++++++++++++ docker/ldd | 1 + renovate.json | 48 ++- src/lib/camoufox/browser.ts | 84 ++++++ src/lib/camoufox/packageManager.ts | 316 ++++++++++++++++++++ src/lib/camoufox/webgl.ts | 45 +++ src/lib/jobs/camoufox.ts | 375 ++++++++++++++++++++++++ src/lib/jobs/potoken.ts | 311 ++++++++++++++------ src/lib/jobs/worker.ts | 81 +++-- src/lib/potoken/attestation.ts | 122 ++++++++ src/lib/potoken/browserPage.ts | 337 +++++++++++++++++++++ src/lib/potoken/constants.ts | 1 + src/lib/potoken/errors.ts | 8 + src/main.ts | 33 ++- src/tests/main_test.ts | 55 ++-- 23 files changed, 2343 insertions(+), 187 deletions(-) create mode 100644 dependencies.json create mode 100644 docker/camoufox-bootstrap.go create mode 100644 docker/ldd create mode 100644 src/lib/camoufox/browser.ts create mode 100644 src/lib/camoufox/packageManager.ts create mode 100644 src/lib/camoufox/webgl.ts create mode 100644 src/lib/jobs/camoufox.ts create mode 100644 src/lib/potoken/attestation.ts create mode 100644 src/lib/potoken/browserPage.ts create mode 100644 src/lib/potoken/constants.ts create mode 100644 src/lib/potoken/errors.ts diff --git a/.github/workflows/deno-check.yaml b/.github/workflows/deno-check.yaml index 8470afb8c..19ab49009 100644 --- a/.github/workflows/deno-check.yaml +++ b/.github/workflows/deno-check.yaml @@ -7,22 +7,43 @@ on: jobs: testing: runs-on: ubuntu-latest + env: + CAMOUFOX_INSTALL_DIR: /var/tmp/youtubei.js/camoufox + DENO_DIR: /var/tmp/youtubei.js/deno-cache + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" steps: - name: Setup repo uses: actions/checkout@v7 - + - name: Setup Deno uses: denoland/setup-deno@v2 with: deno-version: v2.x + - name: Install Camoufox + run: | + playwright_specifier="$(sed '/^[[:space:]]*\/\//d' deno.jsonc | jq -er '.imports["playwright-core"]')" + deno run -A "${playwright_specifier}" install-deps firefox + camoufox_tag="$(jq -er '.camoufox.linux.x86_64.version' dependencies.json)" + camoufox_version="${camoufox_tag#v}" + camoufox_checksum="$(jq -er '.camoufox.linux.x86_64.sha256' dependencies.json)" + mkdir -p /var/tmp/youtubei.js/camoufox + curl -fsSL --output /tmp/camoufox.zip \ + "https://github.com/daijro/camoufox/releases/download/${camoufox_tag}/camoufox-${camoufox_version}-lin.x86_64.zip" + echo "${camoufox_checksum} /tmp/camoufox.zip" | sha256sum -c - + unzip -q /tmp/camoufox.zip -d /var/tmp/youtubei.js/camoufox + printf '{"version":"%s","release":"%s"}\n' \ + "${camoufox_version%%-*}" "${camoufox_version#*-}" \ + > /var/tmp/youtubei.js/camoufox/version.json + rm /tmp/camoufox.zip + - name: Verify formatting run: deno task format - name: Verify typing run: deno task check - + - name: Run linter run: deno task lint @@ -50,7 +71,7 @@ jobs: PROXY_PID=$! sleep 3s curl -s -x http://127.0.0.1:18080 --retry 5 --retry-delay 2 --retry-connrefused https://check.torproject.org/api/ip; echo - rm -rf /var/tmp/youtubei.js + rm -f /var/tmp/youtubei.js/kv_cache.sqlite3* if PROXY=http://127.0.0.1:18080 deno task test; then OPERA_SUCCESS=true kill $PROXY_PID 2>/dev/null @@ -69,7 +90,7 @@ jobs: echo "=== Tor attempt $attempt/12 ===" sudo pkill -HUP tor curl -s --socks5 127.0.0.1:9150 https://check.torproject.org/api/ip; echo - rm -rf /var/tmp/youtubei.js + rm -f /var/tmp/youtubei.js/kv_cache.sqlite3* if PROXY=socks5://127.0.0.1:9150 deno task test; then echo "=== Tests passed with tor ===" exit 0 diff --git a/.github/workflows/release-binaries.yaml b/.github/workflows/release-binaries.yaml index 7f7db420e..a548db079 100644 --- a/.github/workflows/release-binaries.yaml +++ b/.github/workflows/release-binaries.yaml @@ -47,12 +47,15 @@ jobs: - name: Build binary run: | deno compile \ + --self-extracting \ + --include ./dependencies.json \ --include ./src/lib/helpers/youtubePlayerReq.ts \ --include ./src/lib/helpers/getFetchClient.ts \ + --include ./src/lib/jobs/worker.ts \ --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 \ - --allow-net --allow-env --allow-sys=hostname \ - --allow-read=.,/var/tmp/youtubei.js,/tmp/invidious-companion.sock \ - --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock \ + --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi \ + --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock \ + --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock \ --target=${{ matrix.target }} \ --output ${{ steps.binary-name.outputs.name }} \ src/main.ts \ diff --git a/Dockerfile b/Dockerfile index f08fbb650..ea441a161 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,22 +21,23 @@ ARG THC_AMD64_SHA256='cb1797948015da46c222764a99ee30c06a6a9a30f5b87f212a28ea3c6d # we can use these aliases and let dependabot remain simple # inspired by: # https://github.com/dependabot/dependabot-core/issues/2057#issuecomment-1351660410 -FROM alpine:3.24 AS dependabot-alpine FROM debian:13-slim AS dependabot-debian # Retrieve the deno binary from the repository FROM denoland/deno:bin-2.9.2 AS deno-bin +FROM golang:1.25-bookworm AS go-builder +# Build the shell-free volume bootstrap used by the final image. +FROM go-builder AS bootstrap-builder +WORKDIR /src +COPY ./docker/camoufox-bootstrap.go ./main.go +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /camoufox-bootstrap ./main.go -# Stage for creating the non-privileged user -FROM dependabot-alpine AS user-stage - -RUN adduser -u 10001 -S appuser # Stage for downloading files using curl from Debian FROM dependabot-debian AS debian-curl RUN DEBIAN_FRONTEND='noninteractive' && export DEBIAN_FRONTEND && \ - apt-get update && apt-get install -y curl xz-utils + apt-get update && apt-get install -y curl jq xz-utils # Download tiny-health-checker from GitHub FROM debian-curl AS thc-download @@ -86,6 +87,22 @@ ARG TINI_VERSION ENV TINI_VERSION="${TINI_VERSION}" COPY --from=tini-download /tini /tini +# Download a reproducible Camoufox build for the target architecture. +FROM debian-curl AS camoufox-download +COPY ./dependencies.json /dependencies.json +RUN arch="$(dpkg --print-architecture)" && \ + case "${arch}" in \ + amd64) camoufox_arch='x86_64' ;; \ + arm64) camoufox_arch='arm64' ;; \ + *) echo "Unsupported Camoufox architecture: ${arch}" >&2; exit 1 ;; \ + esac && \ + camoufox_tag="$(jq -er --arg arch "${camoufox_arch}" '.camoufox.linux[$arch].version' /dependencies.json)" && \ + camoufox_version="${camoufox_tag#v}" && \ + checksum="$(jq -er --arg arch "${camoufox_arch}" '.camoufox.linux[$arch].sha256' /dependencies.json)" && \ + curl -fsSL --output /camoufox.zip \ + "https://github.com/daijro/camoufox/releases/download/${camoufox_tag}/camoufox-${camoufox_version}-lin.${camoufox_arch}.zip" && \ + echo "${checksum} /camoufox.zip" | sha256sum -c - + # Stage for using git from Debian FROM dependabot-debian AS debian-git RUN DEBIAN_FRONTEND='noninteractive' && export DEBIAN_FRONTEND && \ @@ -114,6 +131,7 @@ WORKDIR /app COPY deno.lock ./ COPY deno.jsonc ./ +COPY dependencies.json ./ COPY ./src/ ./src/ @@ -124,41 +142,89 @@ RUN --mount=type=bind,rw,source=.git,target=/app/.git \ --mount=type=cache,target="${DENO_DIR}" \ deno task compile -FROM gcr.io/distroless/cc AS app +FROM dependabot-debian AS camoufox-runtime -# Copy group file for the non-privileged user from the user-stage -COPY --from=user-stage /etc/group /etc/group - -# Copy passwd file for the non-privileged user from the user-stage -COPY --from=user-stage /etc/passwd /etc/passwd +RUN DEBIAN_FRONTEND='noninteractive' && export DEBIAN_FRONTEND && \ + apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + libasound2 \ + libatk1.0-0t64 \ + libcairo-gobject2 \ + libcairo2 \ + libdbus-1-3 \ + libdbus-glib-1-2 \ + libfontconfig1 \ + libfreetype6 \ + libgdk-pixbuf-2.0-0 \ + libglib2.0-0t64 \ + libgtk-3-0t64 \ + libharfbuzz0b \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libx11-6 \ + libx11-xcb1 \ + libxcb-shm0 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxtst6 && \ + rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man + +FROM gcr.io/distroless/cc-debian13:nonroot AS app + +# Camoufox dynamically loads Firefox/GTK libraries. Copy only the prepared +# runtime filesystem into the shell-less final stage. +COPY --from=camoufox-runtime /lib/ /lib/ +COPY --from=camoufox-runtime /usr/lib/ /usr/lib/ +COPY --from=camoufox-runtime /usr/share/ /usr/share/ +COPY --from=camoufox-runtime /etc/fonts/ /etc/fonts/ +COPY --from=camoufox-runtime /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=camoufox-runtime /var/cache/fontconfig/ /var/cache/fontconfig/ COPY --from=thc-bin /thc /thc COPY --from=tini-bin /tini /tini +COPY --from=bootstrap-builder /camoufox-bootstrap /camoufox-bootstrap +COPY --from=camoufox-download --chown=10001:10001 /camoufox.zip /usr/share/invidious-companion/camoufox.zip +COPY --chmod=0444 ./docker/ldd /usr/bin/ldd # Copy cache directory and set correct permissions -COPY --from=builder --chown=appuser:nogroup /var/tmp/youtubei.js /var/tmp/youtubei.js +COPY --from=builder --chown=10001:10001 /var/tmp/youtubei.js /var/tmp/youtubei.js # Set the working directory WORKDIR /app COPY --from=builder /app/invidious_companion ./ +COPY --chmod=0444 ./dependencies.json ./ ARG HOST PORT THC_VERSION THC_PORT_NAME TINI_VERSION EXPOSE "${PORT}/tcp" ENV SERVER_BASE_PATH=/companion \ + CAMOUFOX_INSTALL_DIR=/var/tmp/youtubei.js/camoufox \ + HOME=/var/tmp/youtubei.js/home \ HOST="${HOST}" \ + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 \ PORT="${PORT}" \ + TMPDIR=/var/tmp/youtubei.js/tmp \ THC_PORT_NAME="${THC_PORT_NAME}" \ THC_PATH="/healthz" \ THC_VERSION="${THC_VERSION}" \ - TINI_VERSION="${TINI_VERSION}" + TINI_VERSION="${TINI_VERSION}" \ + XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache \ + XDG_DATA_HOME=/var/tmp/youtubei.js/xdg-data COPY ./config/ ./config/ # Switch to non-privileged user -USER appuser +USER 10001:10001 -ENTRYPOINT ["/tini", "--", "/app/invidious_companion"] +ENTRYPOINT ["/tini", "--", "/camoufox-bootstrap"] +CMD ["/app/invidious_companion"] HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=5 CMD ["/thc"] diff --git a/README.md b/README.md index 679d6ec55..68256f68e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,15 @@ Companion for Invidious which handle all the video stream retrieval from YouTube ### Requirements -- [deno](https://docs.deno.com/runtime/) +- [deno](https://docs.deno.com/runtime/) +- [Camoufox](https://camoufox.com/) for browser-backed PO token generation. + The Docker image installs a pinned headless browser into the existing + `/var/tmp/youtubei.js` volume automatically. Local development downloads it + into the same directory on first startup. + + Companion falls back to its previous JSDOM generator when Camoufox cannot be + installed or launched. Camoufox runs with `headless: true`; Xvfb is not used. + The production image uses a shell-less Google Distroless runtime. ### Run Locally (development) diff --git a/deno.jsonc b/deno.jsonc index 80001911e..d05aab413 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -2,11 +2,11 @@ // Deno tasks definitions. "tasks": { // Run the server in development mode with file-watching. - "dev": "deno run --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname --allow-read=.,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --watch src/main.ts", + "dev": "DENO_DIR=/var/tmp/youtubei.js/deno-cache XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache deno run --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --watch src/main.ts", // Compile a self-contained executable binary. - "compile": "deno compile --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-read --allow-sys=hostname --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock src/main.ts --_version_date=\"$(git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g)\" --_version_commit=\"$(git rev-list HEAD --max-count=1 --abbrev-commit)\"", + "compile": "deno compile --self-extracting --include ./dependencies.json --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --include ./src/lib/jobs/worker.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock src/main.ts --_version_date=\"$(git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g)\" --_version_commit=\"$(git rev-list HEAD --max-count=1 --abbrev-commit)\"", // Run integration tests. - "test": "deno test --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname --allow-read=.,/var/tmp/youtubei.js,/tmp/invidious-companion.sock --allow-write=/var/tmp/youtubei.js", + "test": "DENO_DIR=/var/tmp/youtubei.js/deno-cache XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache deno test --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock", // Check code formatting. "format": "deno fmt --check src/**", // Type-check the source code without emitting output. @@ -29,7 +29,11 @@ "youtubei.js/NavigationEndpoint": "https://cdn.jsdelivr.net/gh/LuanRT/YouTube.js@v17.2.0-deno/deno/src/parser/classes/NavigationEndpoint.ts", "youtubei.js/PlayerCaptionsTracklist": "https://cdn.jsdelivr.net/gh/LuanRT/YouTube.js@v17.2.0-deno/deno/src/parser/classes/PlayerCaptionsTracklist.ts", "youtubei.js/TabbedFeed": "https://cdn.jsdelivr.net/gh/LuanRT/YouTube.js@v17.2.0-deno/deno/src/core/mixins/TabbedFeed.ts", - // DOM implementation for Node/Deno (used by YouTube.js). + // Headless anti-detect Firefox used for browser-backed PO tokens. + "camoufox-js": "npm:camoufox-js@0.12.0", + // Camoufox's supported browser automation peer. + "playwright-core": "npm:playwright-core@1.60.0", + // DOM fallback used when Camoufox cannot be installed or launched. "jsdom": "npm:jsdom@26.1.0", // Botguard utilities for PO token generation. "bgutils": "https://esm.sh/bgutils-js@3.2.0", diff --git a/deno.lock b/deno.lock index d3d8ff538..aeb08f3b8 100644 --- a/deno.lock +++ b/deno.lock @@ -17,9 +17,11 @@ "jsr:@std/path@^1.1.5": "1.1.6", "jsr:@std/toml@1.0.11": "1.0.11", "npm:@bufbuild/protobuf@2": "2.12.1", + "npm:camoufox-js@0.12.0": "0.12.0_playwright-core@1.60.0", "npm:fflate@0.8.3": "0.8.3", "npm:jsdom@26.1.0": "26.1.0", - "npm:meriyah@7.3.0": "7.3.0" + "npm:meriyah@7.3.0": "7.3.0", + "npm:playwright-core@1.60.0": "1.60.0" }, "jsr": { "@hono/hono@4.12.32": { @@ -89,14 +91,14 @@ } }, "npm": { - "@asamuzakjp/css-color@3.2.0_@csstools+css-parser-algorithms@3.0.5__@csstools+css-tokenizer@3.0.4_@csstools+css-tokenizer@3.0.4": { + "@asamuzakjp/css-color@3.2.0": { "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", "dependencies": [ "@csstools/css-calc", "@csstools/css-color-parser", "@csstools/css-parser-algorithms", "@csstools/css-tokenizer", - "lru-cache" + "lru-cache@10.4.3" ] }, "@bufbuild/protobuf@2.12.1": { @@ -130,9 +132,82 @@ "@csstools/css-tokenizer@3.0.4": { "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==" }, + "@sindresorhus/is@4.6.0": { + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==" + }, + "adm-zip@0.6.0": { + "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==" + }, "agent-base@7.1.4": { "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==" }, + "ansi-regex@5.0.1": { + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "balanced-match@4.0.4": { + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==" + }, + "baseline-browser-mapping@2.11.10": { + "integrity": "sha512-35JEvJ5/KKlbCHjMCsONI2w6HE88STjVdHk+C7d8LtcFxUjZR1KeLP9izofn2qs0KUxX5r4z73bwH/rd+JHacw==", + "bin": true + }, + "better-sqlite3@13.0.2": { + "integrity": "sha512-jW6oufeDhXZaiX9Lw5A+oerVClx4iFrI6uDj1zu7SqUAjak9vbJvA0NEcKLNxHiQHb6kYCoFzzXYV0YOauhV3g==", + "dependencies": [ + "node-addon-api" + ] + }, + "brace-expansion@5.0.9": { + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "dependencies": [ + "balanced-match" + ] + }, + "browserslist@4.28.7": { + "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==", + "dependencies": [ + "baseline-browser-mapping", + "caniuse-lite", + "electron-to-chromium", + "node-releases", + "update-browserslist-db" + ], + "bin": true + }, + "callsites@3.1.0": { + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==" + }, + "camoufox-js@0.12.0_playwright-core@1.60.0": { + "integrity": "sha512-WvWrSL6IH7NixrmB4PM1NODx9cY2R+V/jv9jrmHMVlR4IHP3uheP6whhv0xsoVhFlKIErky5aYx7uA7oECA/Tg==", + "dependencies": [ + "adm-zip", + "better-sqlite3", + "cli-progress", + "commander", + "fingerprint-generator", + "glob", + "impit", + "language-tags", + "maxmind", + "playwright-core", + "pretty-bytes", + "ua-parser-js", + "xml2js" + ], + "bin": true + }, + "caniuse-lite@1.0.30001806": { + "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==" + }, + "cli-progress@3.12.0": { + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "dependencies": [ + "string-width" + ] + }, + "commander@14.0.3": { + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==" + }, "cssstyle@4.6.0": { "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", "dependencies": [ @@ -156,12 +231,62 @@ "decimal.js@10.6.0": { "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==" }, + "detect-europe-js@0.1.2": { + "integrity": "sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==" + }, + "dot-prop@6.0.1": { + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "dependencies": [ + "is-obj" + ] + }, + "electron-to-chromium@1.5.399": { + "integrity": "sha512-lEcqhErbHjXRvd41rnWLpzbyU/IXfIYo7QwaFWmxGeLiLyY2TBCdHnWY88vB+p3ubnihRypDm66panXl7TylLA==" + }, + "emoji-regex@8.0.0": { + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, "entities@6.0.1": { "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==" }, + "escalade@3.2.0": { + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==" + }, "fflate@0.8.3": { "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==" }, + "fingerprint-generator@2.1.87": { + "integrity": "sha512-/VB0SUKbHbguk4fouslD8QVQM3MFKoDiuc+qP2qcmfJkC3gwA71tt1Al42bwLKZyznvP9ZayOpY5iCI0kCWRIg==", + "dependencies": [ + "generative-bayesian-network", + "header-generator", + "tslib" + ] + }, + "generative-bayesian-network@2.1.87": { + "integrity": "sha512-Bd14uZSkGGef6gWsJp170xnTMxeMENcnkJega4UjFrgF0KHSG8IOYA5hgFVOIgWlfyYUy9B2qiLGvXsKR/TlYA==", + "dependencies": [ + "adm-zip", + "tslib" + ] + }, + "glob@13.0.6": { + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dependencies": [ + "minimatch", + "minipass", + "path-scurry" + ] + }, + "header-generator@2.1.87": { + "integrity": "sha512-J3MG508JWaH3UJtHlPM3xaOQ2Iqys8BiW07t6XehzP+HuvVX4MP4C2YztJKasDmB8R3yPwpBfyroOiEi4Ymn6Q==", + "dependencies": [ + "browserslist", + "generative-bayesian-network", + "ow", + "tslib" + ] + }, "html-encoding-sniffer@4.0.0": { "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", "dependencies": [ @@ -188,9 +313,71 @@ "safer-buffer" ] }, + "impit-darwin-arm64@0.14.3": { + "integrity": "sha512-kMoQB+CR+a954pnhe4kf1O2RyJCsqGRE95jIXfgqNOiIMS5O1z0cOMzG/sohJk/nodZVSJ5VdWEV4BKhRWPFSA==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "impit-darwin-x64@0.14.3": { + "integrity": "sha512-ft9+kjz1pR8H5xbbf5U1EgwaaIea5iXHxBf2Q3NoinPpiV7KgyzYSr83pCSrAY6evWk+smG9shHgzhQtgbj1Rg==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "impit-linux-arm64-gnu@0.14.3": { + "integrity": "sha512-+mwta93S6Ndfpca3DDblvrqV1g8Bg6gZFOlEiJHfGJZpiUIQE+A/Pjg2e+inV+WrTtlnLsvff8lgwFvkIGnRug==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "impit-linux-arm64-musl@0.14.3": { + "integrity": "sha512-wYhfH1J8laWkpSN2SAFCKx9npY4vXrk23ex+tOzTGf2xBBqIhpMUff2wa6dOFb0or6EhZHBssoRf0HK3h/htXQ==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "impit-linux-x64-gnu@0.14.3": { + "integrity": "sha512-hAYnutJDGQO5bPvTPKBBFkHgbB7QL7QLDse8c4s21VSzC/EqGliSS3UqR5ZUUwaFdm3t8DJsXOr+gboAUwDR7w==", + "os": ["linux"], + "cpu": ["x64"] + }, + "impit-linux-x64-musl@0.14.3": { + "integrity": "sha512-wXBeHiqCjuyqxg4u5eY4OBNVzsS4Karz/ms/0mjB8aqVZLNER1NaUTeE4J3eZCdytFaJRaAeQvKTH8HfjpPsQg==", + "os": ["linux"], + "cpu": ["x64"] + }, + "impit-win32-arm64-msvc@0.14.3": { + "integrity": "sha512-XiF8MYpm4tF8TMbtsWrcEDDs6NxRNeirfiHM/AyKcr2jiN34hZwbZDM0EseD30PDwb2Pny2JWB+zTxpE1P5CZg==", + "os": ["win32"], + "cpu": ["arm64"] + }, + "impit-win32-x64-msvc@0.14.3": { + "integrity": "sha512-HIaLSpU5SGVx3UDH/R3ZaGMURVgyRVjmalqXj65ABxVUqfXiBo1t5ZObDPPjqvLT3klPnrya1+xF1DQ4dfT+og==", + "os": ["win32"], + "cpu": ["x64"] + }, + "impit@0.14.3": { + "integrity": "sha512-SbCLoeW0YDRox5kQoy71jtpjZE+BwjWO411OdgfbLMbNmSX79s5Y0Y2vCE7AO6zQSq9F4Fu8SJTCKnp8rXQzpg==", + "optionalDependencies": [ + "impit-darwin-arm64", + "impit-darwin-x64", + "impit-linux-arm64-gnu", + "impit-linux-arm64-musl", + "impit-linux-x64-gnu", + "impit-linux-x64-musl", + "impit-win32-arm64-msvc", + "impit-win32-x64-msvc" + ] + }, + "is-fullwidth-code-point@3.0.0": { + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "is-obj@2.0.0": { + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" + }, "is-potential-custom-element-name@1.0.1": { "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" }, + "is-standalone-pwa@0.1.1": { + "integrity": "sha512-9Cbovsa52vNQCjdXOzeQq5CnCbAcRk05aU62K20WO372NrTv0NxibLFCK6lQ4/iZEFdEA3p3t2VNOn8AJ53F5g==" + }, "jsdom@26.1.0": { "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", "dependencies": [ @@ -216,17 +403,68 @@ "xml-name-validator" ] }, + "language-subtag-registry@0.3.23": { + "integrity": "sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==" + }, + "language-tags@2.1.0": { + "integrity": "sha512-D4CgpyCt+61f6z2jHjJS1OmZPviAWM57iJ9OKdFFWSNgS7Udj9QVWqyGs/cveVNF57XpZmhSvMdVIV5mjLA7Vg==", + "dependencies": [ + "language-subtag-registry" + ] + }, + "lodash.isequal@4.5.0": { + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": true + }, "lru-cache@10.4.3": { "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, + "lru-cache@11.5.2": { + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==" + }, + "maxmind@5.0.7": { + "integrity": "sha512-+w637dwfv01MKjkrp4sKDBTEKHLPvWLYb647QTjiz3wG/teSemqudIKNShaS6eqZ7ffxC9oZlQQgIqY0rGojog==", + "dependencies": [ + "mmdb-lib", + "tiny-lru" + ] + }, "meriyah@7.3.0": { "integrity": "sha512-TFAu09uiyoLlx/aEhIZGmlDmycB8Orzml2lpNzwRhLip4a5WJiCxOEoQ8mF6zNhfGqbuI7baS+ZV/0Q9YUBzvw==" }, + "minimatch@10.2.6": { + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dependencies": [ + "brace-expansion" + ] + }, + "minipass@7.1.3": { + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==" + }, + "mmdb-lib@3.0.3": { + "integrity": "sha512-xQPoBXcNjjHiOvOraFBKtA++uNWF6aCVHL9dRKFXEov8eI3QJwtgiw3qApsonFT5SpoqsEVISUTg3HIDs2DiXw==" + }, "ms@2.1.3": { "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "nwsapi@2.2.22": { - "integrity": "sha512-ujSMe1OWVn55euT1ihwCI1ZcAaAU3nxUiDwfDQldc51ZXaB9m2AyOn6/jh1BLe2t/G8xd6uKG1UBF2aZJeg2SQ==" + "node-addon-api@8.9.1": { + "integrity": "sha512-4eUQWVPCUUUiBjLnHS3cXWeC6ryoPUc0U3rP7IuzapoGbzMqd/r6KKO0clr0b+snQhsrueFEhCZDdK+LK7hxKg==" + }, + "node-releases@2.0.51": { + "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==" + }, + "nwsapi@2.2.24": { + "integrity": "sha512-7YRhZ3jS45LwmSCT4b2sVFHt/WuovaktDU07QrtOBY2PXskss5a9jfmR9jptyumwXST+rFjrmppMY1KT/yn35A==" + }, + "ow@0.28.2": { + "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==", + "dependencies": [ + "@sindresorhus/is", + "callsites", + "dot-prop", + "lodash.isequal", + "vali-date" + ] }, "parse5@7.3.0": { "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", @@ -234,6 +472,23 @@ "entities" ] }, + "path-scurry@2.0.2": { + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dependencies": [ + "lru-cache@11.5.2", + "minipass" + ] + }, + "picocolors@1.1.1": { + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "playwright-core@1.60.0": { + "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", + "bin": true + }, + "pretty-bytes@7.1.1": { + "integrity": "sha512-X+vn9z8nOFZQlxOLmfJ0iKDdMD7jYTsTW12OAlCpdoE3Igik6L37pugIZi+N3usuyp5McfgKPWi12q3zvHLeGQ==" + }, "punycode@2.3.1": { "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==" }, @@ -243,15 +498,35 @@ "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sax@1.6.1": { + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==" + }, "saxes@6.0.0": { "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", "dependencies": [ "xmlchars" ] }, + "string-width@4.2.3": { + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": [ + "emoji-regex", + "is-fullwidth-code-point", + "strip-ansi" + ] + }, + "strip-ansi@6.0.1": { + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": [ + "ansi-regex" + ] + }, "symbol-tree@3.2.4": { "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, + "tiny-lru@13.0.0": { + "integrity": "sha512-xDHxKKS1FdF0Tv2P+QT7IeSEg74K/8cEDzbv3Tv6UyHHUgBOjOiQiBp818MGj66dhurQus/IBcoAbwIKtSGc6Q==" + }, "tldts-core@6.1.86": { "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==" }, @@ -274,6 +549,33 @@ "punycode" ] }, + "tslib@2.8.1": { + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "ua-is-frozen@0.1.2": { + "integrity": "sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw==" + }, + "ua-parser-js@2.0.10": { + "integrity": "sha512-t+3Ktbq0Ies2vaSezfOaWiolH4OigQIO1dk+1xDpOydB1COVPocVYOrEV5rqZ0kFY9XYG1v9LutCyMgYBpABcw==", + "dependencies": [ + "detect-europe-js", + "is-standalone-pwa", + "ua-is-frozen" + ], + "bin": true + }, + "update-browserslist-db@1.2.3_browserslist@4.28.7": { + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dependencies": [ + "browserslist", + "escalade", + "picocolors" + ], + "bin": true + }, + "vali-date@1.0.0": { + "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==" + }, "w3c-xmlserializer@5.0.0": { "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", "dependencies": [ @@ -300,12 +602,22 @@ "webidl-conversions" ] }, - "ws@8.18.3": { - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==" + "ws@8.21.1": { + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==" }, "xml-name-validator@5.0.0": { "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==" }, + "xml2js@0.6.2": { + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "dependencies": [ + "sax", + "xmlbuilder" + ] + }, + "xmlbuilder@11.0.1": { + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "xmlchars@2.2.0": { "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" } @@ -5830,9 +6142,11 @@ "jsr:@std/fs@1.0.24", "jsr:@std/path@1.1.6", "jsr:@std/toml@1.0.11", + "npm:camoufox-js@0.12.0", "npm:fflate@0.8.3", "npm:jsdom@26.1.0", - "npm:meriyah@7.3.0" + "npm:meriyah@7.3.0", + "npm:playwright-core@1.60.0" ] } } diff --git a/dependencies.json b/dependencies.json new file mode 100644 index 000000000..e3ce0934d --- /dev/null +++ b/dependencies.json @@ -0,0 +1,14 @@ +{ + "camoufox": { + "linux": { + "x86_64": { + "version": "v152.0.4-beta.28", + "sha256": "924f3109ccd6d47cd6a0384d67a345fadf975d48b6319f8dbbd5954c588982bd" + }, + "arm64": { + "version": "v152.0.4-beta.28", + "sha256": "3a105a2fc929e80a79b4b7fce2c93ed62c4fb2c877f3c1ed2a5d66a1c4fe968f" + } + } + } +} diff --git a/docker-compose.yaml b/docker-compose.yaml index 36cdb77a1..1f1c608b0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,6 +8,7 @@ services: ports: - 127.0.0.1:8282:8282 restart: unless-stopped + shm_size: 256mb cap_drop: - ALL read_only: true diff --git a/docker/camoufox-bootstrap.go b/docker/camoufox-bootstrap.go new file mode 100644 index 000000000..2d87b5804 --- /dev/null +++ b/docker/camoufox-bootstrap.go @@ -0,0 +1,210 @@ +// Shell-free Camoufox installer for the distroless Docker image: it extracts +// the baked-in camoufox.zip into the install directory with no network at +// container start. It mirrors src/lib/camoufox/packageManager.ts (the runtime +// installer used outside Docker). Both read the committed dependencies.json +// for the target version (.camoufox.linux[].version = "vX.Y.Z-release"); +// keep that parsing in lockstep. (version.json is the generated install +// marker, written by install() below.) +package main + +import ( + "archive/zip" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" +) + +const ( + archivePath = "/usr/share/invidious-companion/camoufox.zip" + dependenciesPath = "/app/dependencies.json" + storageRoot = "/var/tmp/youtubei.js" +) + +type version struct { + Version string `json:"version"` + Release string `json:"release"` +} + +type dependencies struct { + Camoufox struct { + Linux map[string]struct { + Version string `json:"version"` + } `json:"linux"` + } `json:"camoufox"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "camoufox bootstrap: %v\n", err) + os.Exit(1) + } +} + +func run() error { + if err := prepareBrowser(); err != nil { + fmt.Fprintf(os.Stderr, "camoufox bootstrap warning: %v; Companion will use its fallback if needed\n", err) + } + if len(os.Args) < 2 { + return fmt.Errorf("no application command was provided") + } + return syscall.Exec(os.Args[1], os.Args[1:], os.Environ()) +} + +func prepareBrowser() error { + installDir := os.Getenv("CAMOUFOX_INSTALL_DIR") + if installDir == "" { + installDir = filepath.Join(storageRoot, "camoufox") + } + installDir = filepath.Clean(installDir) + if !strings.HasPrefix(installDir, storageRoot+string(filepath.Separator)) { + return fmt.Errorf("CAMOUFOX_INSTALL_DIR must be inside %s", storageRoot) + } + + wanted, err := configuredVersion() + if err != nil { + return err + } + + for _, directory := range []string{ + filepath.Join(storageRoot, "home"), + filepath.Join(storageRoot, "tmp"), + filepath.Join(storageRoot, "xdg-cache"), + filepath.Join(storageRoot, "xdg-data"), + } { + if err := os.MkdirAll(directory, 0o755); err != nil { + return err + } + } + + if !isInstalled(installDir, wanted) { + fmt.Printf("Installing Camoufox %s-%s into %s\n", wanted.Version, wanted.Release, installDir) + if err := install(installDir, wanted); err != nil { + return err + } + } + + return nil +} + +func configuredVersion() (version, error) { + architecture := map[string]string{"amd64": "x86_64", "arm64": "arm64"}[runtime.GOARCH] + if architecture == "" { + return version{}, fmt.Errorf("unsupported Camoufox architecture %s", runtime.GOARCH) + } + data, err := os.ReadFile(dependenciesPath) + if err != nil { + return version{}, fmt.Errorf("read dependencies: %w", err) + } + var configured dependencies + if err := json.Unmarshal(data, &configured); err != nil { + return version{}, fmt.Errorf("parse dependencies: %w", err) + } + tag := configured.Camoufox.Linux[architecture].Version + parts := strings.SplitN(strings.TrimPrefix(tag, "v"), "-", 2) + if len(parts) == 2 && parts[0] != "" && parts[1] != "" { + return version{Version: parts[0], Release: parts[1]}, nil + } + return version{}, fmt.Errorf("invalid Camoufox version for %s", architecture) +} + +func isInstalled(installDir string, wanted version) bool { + binary, err := os.Stat(filepath.Join(installDir, "camoufox-bin")) + if err != nil || binary.Mode()&0o111 == 0 { + return false + } + data, err := os.ReadFile(filepath.Join(installDir, "version.json")) + if err != nil { + return false + } + var current version + return json.Unmarshal(data, ¤t) == nil && current == wanted +} + +func install(installDir string, wanted version) error { + // Keep this layout and version.json shape aligned with + // src/lib/camoufox/packageManager.ts. + temporaryDir := fmt.Sprintf("%s.tmp.%d", installDir, os.Getpid()) + if err := os.RemoveAll(temporaryDir); err != nil { + return err + } + defer os.RemoveAll(temporaryDir) + if err := os.MkdirAll(temporaryDir, 0o755); err != nil { + return err + } + if err := extractZip(archivePath, temporaryDir); err != nil { + return err + } + for _, binary := range []string{"camoufox", "camoufox-bin"} { + if err := os.Chmod(filepath.Join(temporaryDir, binary), 0o755); err != nil { + return err + } + } + data, err := json.Marshal(wanted) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(temporaryDir, "version.json"), append(data, '\n'), 0o644); err != nil { + return err + } + if err := os.RemoveAll(installDir); err != nil { + return err + } + return os.Rename(temporaryDir, installDir) +} + +func extractZip(archivePath, destination string) error { + archive, err := zip.OpenReader(archivePath) + if err != nil { + return err + } + defer archive.Close() + + for _, entry := range archive.File { + target := filepath.Join(destination, filepath.FromSlash(entry.Name)) + relative, err := filepath.Rel(destination, target) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("unsafe archive path %q", entry.Name) + } + if entry.FileInfo().IsDir() { + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + if err := extractFile(entry, target); err != nil { + return err + } + } + return nil +} + +func extractFile(entry *zip.File, target string) error { + source, err := entry.Open() + if err != nil { + return err + } + defer source.Close() + + mode := entry.Mode().Perm() + if mode == 0 { + mode = 0o644 + } + destination, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + _, copyErr := io.Copy(destination, source) + closeErr := destination.Close() + if copyErr != nil { + return copyErr + } + return closeErr +} diff --git a/docker/ldd b/docker/ldd new file mode 100644 index 000000000..1a5ef72f9 --- /dev/null +++ b/docker/ldd @@ -0,0 +1 @@ +glibc diff --git a/renovate.json b/renovate.json index 1320df355..5abd5b0ad 100644 --- a/renovate.json +++ b/renovate.json @@ -12,7 +12,53 @@ "minimumReleaseAge": "3 days", "prHourlyLimit": 0, "prConcurrentLimit": 0, + "customDatasources": { + "camoufox-amd64": { + "defaultRegistryUrlTemplate": "https://api.github.com/repos/daijro/camoufox/releases", + "transformTemplates": [ + "{\"releases\": $map($[draft=false], function($release) { ($asset := $release.assets[$contains(name, '-lin.x86_64.zip')][0]; $asset.digest ? {\"version\": $release.tag_name, \"digest\": $substringAfter($asset.digest, 'sha256:'), \"releaseTimestamp\": $release.published_at}) })}" + ] + }, + "camoufox-arm64": { + "defaultRegistryUrlTemplate": "https://api.github.com/repos/daijro/camoufox/releases", + "transformTemplates": [ + "{\"releases\": $map($[draft=false], function($release) { ($asset := $release.assets[$contains(name, '-lin.arm64.zip')][0]; $asset.digest ? {\"version\": $release.tag_name, \"digest\": $substringAfter($asset.digest, 'sha256:'), \"releaseTimestamp\": $release.published_at}) })}" + ] + } + }, + "customManagers": [ + { + "customType": "jsonata", + "fileFormat": "json", + "managerFilePatterns": ["/^dependencies\\.json$/"], + "matchStrings": [ + "{\"currentValue\": camoufox.linux.x86_64.version, \"currentDigest\": camoufox.linux.x86_64.sha256}" + ], + "depNameTemplate": "Camoufox Linux x86_64", + "datasourceTemplate": "custom.camoufox-amd64", + "versioningTemplate": "loose" + }, + { + "customType": "jsonata", + "fileFormat": "json", + "managerFilePatterns": ["/^dependencies\\.json$/"], + "matchStrings": [ + "{\"currentValue\": camoufox.linux.arm64.version, \"currentDigest\": camoufox.linux.arm64.sha256}" + ], + "depNameTemplate": "Camoufox Linux ARM64", + "datasourceTemplate": "custom.camoufox-arm64", + "versioningTemplate": "loose" + } + ], "packageRules": [ + { + "description": "Group Camoufox browser updates", + "matchDatasources": [ + "custom.camoufox-amd64", + "custom.camoufox-arm64" + ], + "groupName": "Camoufox browser" + }, { "description": "Group Deno standard library updates", "matchManagers": ["deno"], @@ -60,4 +106,4 @@ "automergeType": "pr" } ] -} \ No newline at end of file +} diff --git a/src/lib/camoufox/browser.ts b/src/lib/camoufox/browser.ts new file mode 100644 index 000000000..37acaa8eb --- /dev/null +++ b/src/lib/camoufox/browser.ts @@ -0,0 +1,84 @@ +import { type Browser, firefox } from "playwright-core"; +import { sampleWebGL } from "./webgl.ts"; + +type FingerprintOperatingSystem = "linux" | "macos" | "windows"; + +const CAMOU_CONFIG_PREFIX = "CAMOU_CONFIG_"; + +export async function launchCamoufox( + operatingSystem: FingerprintOperatingSystem, +): Promise { + const { launchOptions } = await import("camoufox-js"); + const webGLFingerprint = sampleWebGL(toCamoufoxOperatingSystem( + operatingSystem, + )); + const webGl2Enabled = webGLFingerprint.webGl2Enabled; + delete webGLFingerprint.webGl2Enabled; + + // Camoufox 0.12 opens its read-only fingerprint DB in write mode. Skip that + // internal query, then restore the same sampled WebGL config before launch. + // Revalidate the DB path and CAMOU_CONFIG protocol on camoufox-js upgrades. + const options = await launchOptions({ + block_webrtc: true, + block_webgl: true, + enable_cache: false, + exclude_addons: ["UBO"], + geoip: false, + headless: true, + i_know_what_im_doing: true, + os: operatingSystem, + timeout: 60_000, + }); + options.firefoxUserPrefs ??= {}; + options.firefoxUserPrefs["webgl.disabled"] = false; + options.firefoxUserPrefs["webgl.enable-webgl2"] = webGl2Enabled; + options.firefoxUserPrefs["webgl.force-enabled"] = true; + injectFingerprint( + options.env ??= {}, + webGLFingerprint, + operatingSystem === "windows" ? 2_047 : 32_767, + ); + + return await firefox.launch(options); +} + +function injectFingerprint( + environment: Record, + fingerprint: Record, + chunkSize: number, +): void { + const chunks = Object.entries(environment) + .filter(([key]) => key.startsWith(CAMOU_CONFIG_PREFIX)) + .map(([key, value]) => + [ + Number(key.slice(CAMOU_CONFIG_PREFIX.length)), + String(value), + ] as const + ) + .sort(([left], [right]) => left - right); + const config = JSON.parse(chunks.map(([, value]) => value).join("")); + Object.assign(config, fingerprint); + for (const [key] of chunks) { + delete environment[`${CAMOU_CONFIG_PREFIX}${key}`]; + } + + const serialized = JSON.stringify(config); + for ( + let offset = 0, index = 1; + offset < serialized.length; + offset += chunkSize, index++ + ) { + environment[`${CAMOU_CONFIG_PREFIX}${index}`] = serialized.slice( + offset, + offset + chunkSize, + ); + } +} + +function toCamoufoxOperatingSystem( + operatingSystem: FingerprintOperatingSystem, +): "lin" | "mac" | "win" { + if (operatingSystem === "macos") return "mac"; + if (operatingSystem === "windows") return "win"; + return "lin"; +} diff --git a/src/lib/camoufox/packageManager.ts b/src/lib/camoufox/packageManager.ts new file mode 100644 index 000000000..cfc31fc5c --- /dev/null +++ b/src/lib/camoufox/packageManager.ts @@ -0,0 +1,316 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Derived from https://github.com/apify/camoufox-js/blob/master/src/pkgman.ts +// +// Runtime installer used outside Docker (dev / non-container). The Docker +// image instead ships docker/camoufox-bootstrap.go, which installs the same +// browser from a baked-in zip with no network. Both installers read the +// committed dependencies.json for the target version +// (`.camoufox.linux[].version` = "vX.Y.Z-release", plus `.sha256`), so +// keep that parsing in lockstep across the two. (version.json is the generated +// install marker, not a committed file; see installCamoufox below.) + +import { dirname, join } from "@std/path"; +import { Unzip, UnzipInflate } from "fflate"; +import { createHash } from "node:crypto"; + +const CAMOUFOX_REPOSITORY = "daijro/camoufox"; +const DOWNLOAD_RETRIES = 5; +const DEPENDENCIES_URL = new URL("../../../dependencies.json", import.meta.url); + +const ARCHITECTURES: Partial> = { + x86_64: "x86_64", + aarch64: "arm64", +}; + +const OPERATING_SYSTEMS: Partial> = { + darwin: "mac", + linux: "lin", + windows: "win", +}; + +interface GitHubAsset { + name: string; + browser_download_url: string; +} + +interface GitHubRelease { + assets: GitHubAsset[]; +} + +interface CamoufoxVersion { + digest: string; + release: string; + tag: string; + version: string; +} + +interface Dependencies { + camoufox?: { + linux?: Record; + }; +} + +export async function isConfiguredCamoufoxInstalled( + installDirectory: string, +): Promise { + const executable = await Deno.stat(join(installDirectory, "camoufox-bin")); + if (!executable.isFile) return false; + const configured = await readConfiguredVersion(); + const installed = JSON.parse( + await Deno.readTextFile(join(installDirectory, "version.json")), + ) as { version?: string; release?: string }; + return installed.version === configured.version && + installed.release === configured.release; +} + +export async function installCamoufox( + installDirectory: string, +): Promise { + // Keep version.json and extracted layout aligned with + // docker/camoufox-bootstrap.go, which seeds the Distroless volume. + const version = await readConfiguredVersion(); + const url = await getReleaseUrl(version); + const installParent = dirname(installDirectory); + await Deno.mkdir(installParent, { recursive: true }); + const stagingDirectory = await Deno.makeTempDir({ + dir: installParent, + prefix: ".camoufox-install-", + }); + const archivePath = join(stagingDirectory, "camoufox.zip"); + const extractedDirectory = join(stagingDirectory, "browser"); + + try { + console.log("[INFO] Downloading Camoufox", { + version: version.tag, + }); + const digest = await downloadFile(url, archivePath); + if (digest !== version.digest) { + throw new Error( + `Camoufox checksum mismatch: expected ${version.digest}, got ${digest}`, + ); + } + await extractArchive(archivePath, extractedDirectory); + await Deno.writeTextFile( + join(extractedDirectory, "version.json"), + JSON.stringify({ + version: version.version, + release: version.release, + }), + ); + if (Deno.build.os !== "windows") { + await makeExecutable(extractedDirectory); + } + + await Deno.remove(installDirectory, { recursive: true }).catch( + (error) => { + if (!(error instanceof Deno.errors.NotFound)) throw error; + }, + ); + await Deno.rename(extractedDirectory, installDirectory); + console.log("[INFO] Camoufox was installed", { installDirectory }); + } finally { + await Deno.remove(stagingDirectory, { recursive: true }).catch( + () => {}, + ); + } +} + +async function readConfiguredVersion(): Promise { + const architecture = ARCHITECTURES[Deno.build.arch]; + if (!architecture) { + throw new Error(`Camoufox does not support ${Deno.build.arch}`); + } + const dependencies = JSON.parse( + await Deno.readTextFile(DEPENDENCIES_URL), + ) as Dependencies; + const configured = dependencies.camoufox?.linux?.[architecture]; + const match = configured?.version?.match(/^v(\d+(?:\.\d+)+)-(.+)$/); + if (!match) { + throw new Error(`No valid Camoufox version for ${architecture}`); + } + if (!configured?.sha256?.match(/^[a-f0-9]{64}$/)) { + throw new Error(`No valid Camoufox checksum for ${architecture}`); + } + return { + tag: `v${match[1]}-${match[2]}`, + version: match[1], + release: match[2], + digest: configured.sha256, + }; +} + +async function getReleaseUrl(version: CamoufoxVersion): Promise { + const operatingSystem = OPERATING_SYSTEMS[Deno.build.os]; + if (!operatingSystem) { + throw new Error(`Camoufox does not support ${Deno.build.os}`); + } + const architecture = ARCHITECTURES[Deno.build.arch]; + if (!architecture) { + throw new Error(`Camoufox does not support ${Deno.build.arch}`); + } + + const apiUrl = + `https://api.github.com/repos/${CAMOUFOX_REPOSITORY}/releases/tags/${version.tag}`; + const response = await fetchWithRetries(apiUrl); + const release = await response.json() as GitHubRelease; + const assetName = + `camoufox-${version.version}-${version.release}-${operatingSystem}.${architecture}.zip`; + const asset = release.assets.find((candidate) => + candidate.name === assetName + ); + if (!asset) { + throw new Error(`Camoufox release ${version.tag} has no ${assetName}`); + } + return asset.browser_download_url; +} + +async function downloadFile(url: string, destination: string): Promise { + const response = await fetchWithRetries(url); + if (!response.body) throw new Error("Camoufox download had no body"); + const file = await Deno.create(destination); + const digest = createHash("sha256"); + try { + for await (const chunk of response.body) { + digest.update(chunk); + await writeAll(file, chunk); + } + } finally { + file.close(); + } + return digest.digest("hex"); +} + +async function fetchWithRetries(url: string): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= DOWNLOAD_RETRIES; attempt++) { + try { + const response = await fetch(url, { + headers: authorizationHeaders(url), + }); + if (response.ok) return response; + await response.body?.cancel(); + lastError = new Error( + `Camoufox download returned HTTP ${response.status}`, + ); + } catch (error) { + lastError = error; + } + if (attempt < DOWNLOAD_RETRIES) { + await new Promise((resolve) => setTimeout(resolve, 5_000)); + } + } + throw new Error( + `Failed to download ${url} after ${DOWNLOAD_RETRIES} attempts`, + { cause: lastError }, + ); +} + +function authorizationHeaders(url: string): HeadersInit { + const token = Deno.env.get("GITHUB_TOKEN"); + const host = new URL(url).hostname; + return token && (host === "api.github.com" || host === "github.com") + ? { authorization: `Bearer ${token}` } + : {}; +} + +async function extractArchive( + archivePath: string, + destination: string, +): Promise { + await Deno.mkdir(destination, { recursive: true }); + const openFiles = new Set(); + let extractionError: unknown; + const archive = new Unzip((entry) => { + try { + const outputPath = safeArchivePath(destination, entry.name); + if (entry.name.endsWith("/")) { + Deno.mkdirSync(outputPath, { recursive: true }); + entry.ondata = (error) => { + if (error) extractionError = error; + }; + } else { + Deno.mkdirSync(dirname(outputPath), { recursive: true }); + const output = Deno.createSync(outputPath); + openFiles.add(output); + let closed = false; + const closeOutput = () => { + if (closed) return; + closed = true; + output.close(); + openFiles.delete(output); + }; + entry.ondata = (error, data, final) => { + if (closed) return; + if (error) { + extractionError = error; + closeOutput(); + return; + } + try { + writeAllSync(output, data); + } catch (writeError) { + extractionError = writeError; + closeOutput(); + return; + } + if (final) closeOutput(); + }; + } + entry.start(); + } catch (error) { + extractionError = error; + entry.terminate(); + } + }); + archive.register(UnzipInflate); + + try { + const source = await Deno.open(archivePath, { read: true }); + for await (const chunk of source.readable) { + archive.push(chunk); + if (extractionError) throw extractionError; + } + archive.push(new Uint8Array(), true); + if (extractionError) throw extractionError; + } finally { + for (const file of openFiles) file.close(); + } +} + +async function writeAll(file: Deno.FsFile, data: Uint8Array): Promise { + let written = 0; + while (written < data.length) { + written += await file.write(data.subarray(written)); + } +} + +function safeArchivePath(destination: string, entryName: string): string { + const segments = entryName.replaceAll("\\", "/").split("/").filter( + Boolean, + ); + if ( + entryName.startsWith("/") || + segments.some((segment) => segment === "..") + ) { + throw new Error(`Unsafe Camoufox archive path: ${entryName}`); + } + return join(destination, ...segments); +} + +function writeAllSync(file: Deno.FsFile, data: Uint8Array): void { + let written = 0; + while (written < data.length) { + written += file.writeSync(data.subarray(written)); + } +} + +async function makeExecutable(directory: string): Promise { + await Deno.chmod(directory, 0o755); + for await (const entry of Deno.readDir(directory)) { + const path = join(directory, entry.name); + if (entry.isDirectory) await makeExecutable(path); + else await Deno.chmod(path, 0o755); + } +} diff --git a/src/lib/camoufox/webgl.ts b/src/lib/camoufox/webgl.ts new file mode 100644 index 000000000..d598e2be2 --- /dev/null +++ b/src/lib/camoufox/webgl.ts @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Derived from https://github.com/apify/camoufox-js/blob/master/src/webgl/sample.ts + +import { fromFileUrl } from "@std/path"; +import { DatabaseSync } from "node:sqlite"; + +interface WebGLRow { + data: string; + weight: number; +} + +export function sampleWebGL( + operatingSystem: "lin" | "mac" | "win", +): Record & { webGl2Enabled?: boolean } { + const packageUrl = import.meta.resolve("camoufox-js"); + const databasePath = fromFileUrl( + new URL("./data-files/webgl_data.db", packageUrl), + ); + const database = new DatabaseSync(databasePath, { readOnly: true }); + try { + const rows = database.prepare( + `SELECT data, ${operatingSystem} AS weight FROM webgl_fingerprints WHERE ${operatingSystem} > 0`, + ).all() as unknown as WebGLRow[]; + if (rows.length === 0) { + throw new Error( + `No WebGL fingerprints found for ${operatingSystem}`, + ); + } + + const threshold = Math.random() * rows.reduce( + (sum, row) => sum + row.weight, + 0, + ); + let cumulativeWeight = 0; + for (const row of rows) { + cumulativeWeight += row.weight; + if (cumulativeWeight >= threshold) return JSON.parse(row.data); + } + return JSON.parse(rows.at(-1)!.data); + } finally { + database.close(); + } +} diff --git a/src/lib/jobs/camoufox.ts b/src/lib/jobs/camoufox.ts new file mode 100644 index 000000000..dba53e734 --- /dev/null +++ b/src/lib/jobs/camoufox.ts @@ -0,0 +1,375 @@ +import { Innertube } from "youtubei.js"; +import type { Browser, BrowserContext, Page } from "playwright-core"; +import { PLAYER_ID } from "../../constants.ts"; +import { + type FetchFunction, + getAttestationChallenge, + getIntegrityToken, +} from "../potoken/attestation.ts"; +import { + createMinter, + destroyAllGenerationsOnPage, + destroyGenerationOnPage, + initializePoTokenPage, + mintOnPage, + runBotGuard, +} from "../potoken/browserPage.ts"; +import { + installCamoufox, + isConfiguredCamoufoxInstalled, +} from "../camoufox/packageManager.ts"; +import { launchCamoufox } from "../camoufox/browser.ts"; +import { BrowserPoTokenUnavailableError } from "../potoken/errors.ts"; + +export interface CamoufoxPoTokenGeneration { + sessionPoToken: string; + visitorData: string; + userAgent: string; + mint(contentBinding: string): Promise; + activate(): Promise; + discard(): Promise; +} + +class CamoufoxPoTokenRuntime { + private browser?: Browser; + private context?: BrowserContext; + private page?: Page; + private generationIds = new Set(); + private activeGenerationId?: string; + private previousGenerationId?: string; + private mintOperations = new Map>>(); + private operationTail: Promise = Promise.resolve(); + + createGeneration( + fetchImpl: FetchFunction, + cookies: string, + ): Promise { + return this.enqueue(async () => { + const page = await this.ensurePage(); + const generationId = crypto.randomUUID(); + + try { + const userAgent = await page.evaluate(() => + navigator.userAgent + ); + const bootstrapClient = await Innertube.create({ + enable_session_cache: false, + fetch: fetchImpl, + user_agent: userAgent, + retrieve_player: false, + cookie: cookies || undefined, + player_id: PLAYER_ID, + }); + const visitorData = + bootstrapClient.session.context.client.visitorData; + if (!visitorData) { + throw new Error("Could not get visitor data"); + } + + const challenge = await getAttestationChallenge( + bootstrapClient, + fetchImpl, + userAgent, + ); + const botguardResponse = await runBotGuard( + page, + generationId, + challenge, + ); + const integrityToken = await getIntegrityToken( + fetchImpl, + userAgent, + botguardResponse, + ); + await createMinter(page, generationId, integrityToken); + this.generationIds.add(generationId); + + const sessionPoToken = await mintOnPage( + page, + generationId, + visitorData, + ); + let disposed = false; + + return { + sessionPoToken, + visitorData, + userAgent, + mint: (contentBinding) => { + if (disposed) { + return Promise.reject( + new Error("PO token generation was retired"), + ); + } + return this.mint(generationId, contentBinding); + }, + activate: async () => { + if (disposed) { + throw new Error("PO token generation was retired"); + } + await this.activate(generationId); + }, + discard: async () => { + if (disposed) return; + disposed = true; + await this.discard(generationId); + }, + }; + } catch (error) { + await destroyGenerationOnPage(page, generationId).catch( + () => {}, + ); + this.generationIds.delete(generationId); + throw error; + } + }); + } + + close(): Promise { + return this.enqueue(() => this.closeBrowser()); + } + + private async mint( + generationId: string, + contentBinding: string, + ): Promise { + if (!this.generationIds.has(generationId)) { + throw new Error("PO token generation is no longer available"); + } + const page = this.getLivePage(); + const operation = mintOnPage(page, generationId, contentBinding); + const operations = this.mintOperations.get(generationId) ?? new Set(); + operations.add(operation); + this.mintOperations.set(generationId, operations); + try { + return await operation; + } finally { + operations.delete(operation); + if (operations.size === 0) { + this.mintOperations.delete(generationId); + } + } + } + + private activate(generationId: string): Promise { + return this.enqueue(async () => { + if (!this.generationIds.has(generationId)) { + throw new Error( + "Cannot activate an unavailable PO token generation", + ); + } + if (this.activeGenerationId === generationId) return; + + const staleGenerationId = this.previousGenerationId; + this.previousGenerationId = this.activeGenerationId; + this.activeGenerationId = generationId; + + if (staleGenerationId) { + await this.destroyGeneration(staleGenerationId).catch( + (error) => { + console.warn( + "[WARN] Failed to retire old PO token state", + { + error, + }, + ); + }, + ); + } + }); + } + + private discard(generationId: string): Promise { + return this.enqueue(async () => { + if ( + generationId === this.activeGenerationId || + generationId === this.previousGenerationId + ) { + return; + } + await this.destroyGeneration(generationId); + }); + } + + private async destroyGeneration(generationId: string): Promise { + if (!this.generationIds.delete(generationId)) return; + await Promise.allSettled(this.mintOperations.get(generationId) ?? []); + const page = this.page; + if (page && !page.isClosed()) { + await destroyGenerationOnPage(page, generationId); + } + } + + private enqueue(operation: () => Promise): Promise { + const result = this.operationTail.then(operation); + this.operationTail = result.then(() => undefined, () => undefined); + return result; + } + + private getLivePage(): Page { + if (!this.page || this.page.isClosed()) { + throw new Error("Camoufox page is unavailable"); + } + return this.page; + } + + private async ensurePage(): Promise { + if (this.browser?.isConnected() && this.page && !this.page.isClosed()) { + return this.page; + } + + await this.closeBrowser(); + const installDirectory = configureInstallDirectory(); + if (Deno.build.os === "linux" && installDirectory) { + await configureTemporaryDirectory(); + await ensureCamoufoxInstalled(installDirectory); + } + let browser: Browser; + try { + browser = await launchCamoufox(fingerprintOperatingSystem()); + } catch (error) { + throw new BrowserPoTokenUnavailableError( + "Camoufox could not be imported or launched", + { cause: error }, + ); + } + + try { + const context = await browser.newContext({ viewport: null }); + const page = await context.newPage(); + await initializePoTokenPage(page); + + this.browser = browser; + this.context = context; + this.page = page; + browser.once("disconnected", () => this.invalidate(browser, page)); + page.once("crash", () => this.invalidate(browser, page)); + page.once("close", () => this.invalidate(browser, page)); + console.log("[INFO] Camoufox PO token page is ready", { + installDirectory, + }); + return page; + } catch (error) { + await browser.close().catch(() => {}); + throw new BrowserPoTokenUnavailableError( + "Camoufox could not initialize its PO token page", + { cause: error }, + ); + } + } + + private invalidate(browser: Browser, page: Page): void { + if (this.browser !== browser || this.page !== page) return; + this.browser = undefined; + this.context = undefined; + this.page = undefined; + this.generationIds.clear(); + this.activeGenerationId = undefined; + this.previousGenerationId = undefined; + if (browser.isConnected()) void browser.close().catch(() => {}); + } + + private async closeBrowser(): Promise { + const browser = this.browser; + const context = this.context; + const page = this.page; + this.browser = undefined; + this.context = undefined; + this.page = undefined; + this.generationIds.clear(); + this.activeGenerationId = undefined; + this.previousGenerationId = undefined; + + await Promise.allSettled( + [...this.mintOperations.values()].flatMap(( + operations, + ) => [...operations]), + ); + this.mintOperations.clear(); + + if (page && !page.isClosed()) { + await destroyAllGenerationsOnPage(page).catch(() => {}); + } + await context?.close().catch(() => {}); + await browser?.close().catch(() => {}); + } +} + +const runtime = new CamoufoxPoTokenRuntime(); + +export function createCamoufoxPoTokenGeneration( + fetchImpl: FetchFunction, + cookies: string, +): Promise { + return runtime.createGeneration(fetchImpl, cookies); +} + +export function closeCamoufoxPoTokenRuntime(): Promise { + return runtime.close(); +} + +function configureInstallDirectory(): string | undefined { + let installDirectory = Deno.env.get("CAMOUFOX_INSTALL_DIR"); + if (!installDirectory && Deno.build.os === "linux") { + installDirectory = "/var/tmp/youtubei.js/camoufox"; + Deno.env.set("CAMOUFOX_INSTALL_DIR", installDirectory); + } + return installDirectory; +} + +async function configureTemporaryDirectory(): Promise { + if (Deno.env.has("TMPDIR")) return; + const tempDirectory = "/var/tmp/youtubei.js/tmp"; + try { + await Deno.mkdir(tempDirectory, { recursive: true }); + Deno.env.set("TMPDIR", tempDirectory); + } catch (error) { + throw new BrowserPoTokenUnavailableError( + `Camoufox temporary directory could not be created at ${tempDirectory}`, + { cause: error }, + ); + } +} + +async function ensureCamoufoxInstalled( + installDirectory: string, +): Promise { + try { + if (await isConfiguredCamoufoxInstalled(installDirectory)) return; + } catch (error) { + if (browserDownloadDisabled()) { + throw new BrowserPoTokenUnavailableError( + `Camoufox is not installed at ${installDirectory}`, + { cause: error }, + ); + } + } + + console.log("[INFO] Camoufox is not installed; downloading it", { + installDirectory, + }); + try { + await installCamoufox(installDirectory); + + const executable = await Deno.stat( + `${installDirectory}/camoufox-bin`, + ); + if (!executable.isFile) throw new Error("not a file"); + } catch (error) { + throw new BrowserPoTokenUnavailableError( + `Camoufox could not be installed at ${installDirectory}`, + { cause: error }, + ); + } +} + +function browserDownloadDisabled(): boolean { + const value = Deno.env.get("PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD"); + return Boolean(value && value !== "0" && value !== "false"); +} + +function fingerprintOperatingSystem(): "linux" | "macos" | "windows" { + if (Deno.build.os === "darwin") return "macos"; + if (Deno.build.os === "windows") return "windows"; + return "linux"; +} diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts index 8d8fa7d20..465497047 100644 --- a/src/lib/jobs/potoken.ts +++ b/src/lib/jobs/potoken.ts @@ -5,6 +5,12 @@ import { } from "../helpers/youtubePlayerHandling.ts"; import type { Config } from "../helpers/config.ts"; import { Metrics } from "../helpers/metrics.ts"; +import { + closeCamoufoxPoTokenRuntime, + createCamoufoxPoTokenGeneration, +} from "./camoufox.ts"; +import { BrowserPoTokenUnavailableError } from "../potoken/errors.ts"; +import { InputMessage, OutputMessageSchema } from "./worker.ts"; let getFetchClientLocation = "getFetchClient"; if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -18,122 +24,247 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { } const { getFetchClient } = await import(getFetchClientLocation); -import { InputMessage, OutputMessageSchema } from "./worker.ts"; import { PLAYER_ID } from "../../constants.ts"; +export type TokenMinter = (videoId: string) => Promise; +let generationInFlight: Promise | undefined; +let activeFallbackWorker: TokenGeneratorWorker | undefined; +type PoTokenResult = { innertubeClient: Innertube; tokenMinter: TokenMinter }; interface TokenGeneratorWorker extends Omit { postMessage(message: InputMessage): void; } -const workers: TokenGeneratorWorker[] = []; +// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts +export function poTokenGenerate( + config: Config, + metrics: Metrics | undefined, +): Promise { + if (generationInFlight) return generationInFlight; + generationInFlight = generatePoToken(config, metrics).finally(() => { + generationInFlight = undefined; + }); + return generationInFlight; +} -function createMinter(worker: TokenGeneratorWorker) { - return (videoId: string): Promise => { - const { promise, resolve } = Promise.withResolvers(); - // generate a UUID to identify the request as many minter calls - // may be made within a timespan, and this function will be - // informed about all of them until it's got its own - const requestId = crypto.randomUUID(); - const listener = (message: MessageEvent) => { - const parsedMessage = OutputMessageSchema.parse(message.data); - if ( - parsedMessage.type === "content-token" && - parsedMessage.requestId === requestId - ) { - worker.removeEventListener("message", listener); - resolve(parsedMessage.contentToken); - } +async function generatePoToken( + config: Config, + metrics: Metrics | undefined, +): Promise { + const fetchImpl = getFetchClient(config); + let generation; + try { + generation = await createCamoufoxPoTokenGeneration( + fetchImpl, + config.youtube_session.cookies, + ); + } catch (error) { + if (error instanceof BrowserPoTokenUnavailableError) { + console.warn( + "[WARN] Camoufox is unavailable; using the JSDOM PO token fallback", + { error }, + ); + return await generateWithJSDOM(config, metrics); + } + throw error; + } + + try { + const instantiatedInnertubeClient = await Innertube.create({ + enable_session_cache: false, + po_token: generation.sessionPoToken, + visitor_data: generation.visitorData, + user_agent: generation.userAgent, + fetch: fetchImpl, + generate_session_locally: true, + cookie: config.youtube_session.cookies || undefined, + player_id: PLAYER_ID, + }); + await checkToken({ + instantiatedInnertubeClient, + config, + integrityTokenBasedMinter: generation.mint, + metrics, + }); + await generation.activate(); + if (activeFallbackWorker) { + void shutdownWorker(activeFallbackWorker); + activeFallbackWorker = undefined; + } + console.log("[INFO] Successfully generated PO token with Camoufox"); + return { + innertubeClient: instantiatedInnertubeClient, + tokenMinter: generation.mint, }; - worker.addEventListener("message", listener); - worker.postMessage({ - type: "content-token-request", - videoId, - requestId, + } catch (error) { + await generation.discard(); + console.log("[WARN] Failed to get valid PO token, will retry", { + error, }); - - return promise; - }; + throw error; + } } -export type TokenMinter = ReturnType; +export async function closePoTokenRuntime(): Promise { + const fallbackWorker = activeFallbackWorker; + activeFallbackWorker = undefined; + await Promise.all([ + fallbackWorker ? shutdownWorker(fallbackWorker) : Promise.resolve(), + closeCamoufoxPoTokenRuntime(), + ]); +} -// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts -export const poTokenGenerate = ( +function generateWithJSDOM( config: Config, metrics: Metrics | undefined, -): Promise<{ innertubeClient: Innertube; tokenMinter: TokenMinter }> => { - const { promise, resolve, reject } = Promise.withResolvers< - Awaited> - >(); - +): Promise { + const { promise, resolve, reject } = Promise.withResolvers(); const worker: TokenGeneratorWorker = new Worker( new URL("./worker.ts", import.meta.url).href, - { - type: "module", - name: "PO Token Generator", - }, + { type: "module", name: "JSDOM PO Token Fallback" }, ); - // take note of the worker so we can kill it once a new one takes its place - workers.push(worker); - worker.addEventListener("message", async (event) => { - const parsedMessage = OutputMessageSchema.parse(event.data); + let settled = false; + const timeout = setTimeout( + () => fail(new Error("JSDOM PO token initialization timed out")), + 5 * 60_000, + ); + const fail = (error: unknown) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + worker.terminate(); + reject(error); + }; - // worker is listening for messages - if (parsedMessage.type === "ready") { - const untypedPostMessage = worker.postMessage.bind(worker); - worker.postMessage = (message: InputMessage) => - untypedPostMessage(message); + worker.addEventListener("error", (event) => { + fail(event.error ?? new Error(event.message)); + }); + worker.addEventListener("messageerror", () => { + fail(new Error("Could not deserialize JSDOM worker message")); + }); + worker.addEventListener("message", async (event) => { + const result = OutputMessageSchema.safeParse(event.data); + if (!result.success) { + fail(new Error("Invalid JSDOM worker response")); + return; + } + const message = result.data; + if (message.type === "ready") { worker.postMessage({ type: "initialise", config }); + return; } - - if (parsedMessage.type === "error") { - console.log({ errorFromWorker: parsedMessage.error }); - worker.terminate(); - reject(parsedMessage.error); + if (message.type === "error") { + fail(new Error(message.error)); + return; } + if (message.type !== "initialised") return; - // worker is initialised and has passed back a session token and visitor data - if (parsedMessage.type === "initialised") { - try { - const instantiatedInnertubeClient = await Innertube.create({ - enable_session_cache: false, - po_token: parsedMessage.sessionPoToken, - visitor_data: parsedMessage.visitorData, - fetch: getFetchClient(config), - generate_session_locally: true, - cookie: config.youtube_session.cookies || undefined, - player_id: PLAYER_ID, - }); - const minter = createMinter(worker); - // check token from minter - await checkToken({ - instantiatedInnertubeClient, - config, - integrityTokenBasedMinter: minter, - metrics, - }); - console.log("[INFO] Successfully generated PO token"); - const numberToKill = workers.length - 1; - for (let i = 0; i < numberToKill; i++) { - const workerToKill = workers.shift(); - workerToKill?.terminate(); - } - return resolve({ - innertubeClient: instantiatedInnertubeClient, - tokenMinter: minter, - }); - } catch (err) { - console.log("[WARN] Failed to get valid PO token, will retry", { - err, - }); - worker.terminate(); - reject(err); + try { + const instantiatedInnertubeClient = await Innertube.create({ + enable_session_cache: false, + po_token: message.sessionPoToken, + visitor_data: message.visitorData, + fetch: getFetchClient(config), + generate_session_locally: true, + cookie: config.youtube_session.cookies || undefined, + player_id: PLAYER_ID, + }); + const minter = createJSDOMMinter(worker); + await checkToken({ + instantiatedInnertubeClient, + config, + integrityTokenBasedMinter: minter, + metrics, + }); + if (settled) return; + settled = true; + clearTimeout(timeout); + const previousWorker = activeFallbackWorker; + activeFallbackWorker = worker; + if (previousWorker && previousWorker !== worker) { + void shutdownWorker(previousWorker); } + console.log("[INFO] Successfully generated PO token with JSDOM"); + resolve({ + innertubeClient: instantiatedInnertubeClient, + tokenMinter: minter, + }); + } catch (error) { + fail(error); } }); - return promise; -}; +} + +function createJSDOMMinter(worker: TokenGeneratorWorker): TokenMinter { + return (videoId) => { + const { promise, resolve, reject } = Promise.withResolvers(); + const requestId = crypto.randomUUID(); + const timeout = setTimeout(() => { + cleanup(); + reject(new Error(`PO token mint timed out for video ${videoId}`)); + }, 30_000); + const cleanup = () => { + clearTimeout(timeout); + worker.removeEventListener("message", listener); + worker.removeEventListener("error", errorListener); + }; + const errorListener = (event: ErrorEvent) => { + cleanup(); + reject(event.error ?? new Error(event.message)); + }; + const listener = (event: MessageEvent) => { + const result = OutputMessageSchema.safeParse(event.data); + if (!result.success) return; + const message = result.data; + if ( + message.type === "content-token" && + message.requestId === requestId + ) { + cleanup(); + resolve(message.contentToken); + } else if ( + message.type === "content-token-error" && + message.requestId === requestId + ) { + cleanup(); + reject(new Error(message.error)); + } + }; + worker.addEventListener("message", listener); + worker.addEventListener("error", errorListener); + worker.postMessage({ + type: "content-token-request", + videoId, + requestId, + }); + return promise; + }; +} + +function shutdownWorker(worker: TokenGeneratorWorker): Promise { + return new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + clearTimeout(timeout); + worker.removeEventListener("message", listener); + worker.terminate(); + resolve(); + }; + const timeout = setTimeout(finish, 5_000); + const listener = (event: MessageEvent) => { + const result = OutputMessageSchema.safeParse(event.data); + if (result.success && result.data.type === "shutdown") finish(); + }; + worker.addEventListener("message", listener); + try { + worker.postMessage({ type: "shutdown" }); + } catch { + finish(); + } + }); +} async function checkToken({ instantiatedInnertubeClient, diff --git a/src/lib/jobs/worker.ts b/src/lib/jobs/worker.ts index 6da6fd793..7c7edc9e4 100644 --- a/src/lib/jobs/worker.ts +++ b/src/lib/jobs/worker.ts @@ -7,6 +7,7 @@ import type { WebPoSignalOutput } from "bgutils"; import { JSDOM } from "jsdom"; import { Innertube } from "youtubei.js"; import { PLAYER_ID } from "../../constants.ts"; +import { INTEGRITY_TOKEN_REQUEST_KEY } from "../potoken/constants.ts"; let getFetchClientLocation = "getFetchClient"; if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -37,9 +38,11 @@ const InputContentTokenSchema = z.object({ }).strict(); export type InputInitialise = z.infer; export type InputContentToken = z.infer; +const InputShutdownSchema = z.object({ type: z.literal("shutdown") }).strict(); const InputMessageSchema = z.union([ InputInitialiseSchema, InputContentTokenSchema, + InputShutdownSchema, ]); export type InputMessage = z.infer; @@ -60,15 +63,24 @@ const OutputContentTokenSchema = z.object({ requestId: InputContentTokenSchema.shape.requestId, }).strict(); +const OutputContentTokenErrorSchema = z.object({ + type: z.literal("content-token-error"), + error: z.string(), + requestId: InputContentTokenSchema.shape.requestId, +}).strict(); + const OutputErrorSchema = z.object({ type: z.literal("error"), - error: z.any(), + error: z.string(), }).strict(); +const OutputShutdownSchema = z.object({ type: z.literal("shutdown") }).strict(); export const OutputMessageSchema = z.union([ OutputReadySchema, OutputInitialiseSchema, OutputContentTokenSchema, + OutputContentTokenErrorSchema, OutputErrorSchema, + OutputShutdownSchema, ]); type OutputMessage = z.infer; @@ -76,14 +88,16 @@ const IntegrityTokenResponse = z.tuple([z.string()]).rest(z.any()); const isWorker = typeof WorkerGlobalScope !== "undefined" && self instanceof WorkerGlobalScope; + if (isWorker) { // helper function to force type-checking - const untypedPostmessage = self.postMessage.bind(self); + const untypedPostMessage = self.postMessage.bind(self); const postMessage = (message: OutputMessage) => { - untypedPostmessage(message); + untypedPostMessage(message); }; - let minter: BG.WebPoMinter; + let minter: BG.WebPoMinter | undefined; + let cleanup: (() => void) | undefined; onmessage = async (event) => { const message = InputMessageSchema.parse(event.data); @@ -96,37 +110,50 @@ if (isWorker) { sessionPoToken, visitorData, generatedMinter, + cleanup: generatedCleanup, } = await setup({ fetchImpl, innertubeClientCookies: message.config.youtube_session.cookies, }); minter = generatedMinter; + cleanup = generatedCleanup; postMessage({ type: "initialised", sessionPoToken, visitorData, }); - } catch (err) { - postMessage({ type: "error", error: err }); + } catch (error) { + postMessage({ type: "error", error: errorMessage(error) }); } + return; } + // this is called every time a video needs a content token if (message.type === "content-token-request") { - if (!minter) { - throw new Error( - "Minter not yet ready, must initialise first", - ); + try { + if (!minter) { + throw new Error("PO token minter is not initialized"); + } + postMessage({ + type: "content-token", + contentToken: await minter.mintAsWebsafeString( + message.videoId, + ), + requestId: message.requestId, + }); + } catch (error) { + postMessage({ + type: "content-token-error", + error: errorMessage(error), + requestId: message.requestId, + }); } - const contentToken = await minter.mintAsWebsafeString( - message.videoId, - ); - postMessage({ - type: "content-token", - contentToken, - requestId: message.requestId, - }); + return; } + + cleanup?.(); + postMessage({ type: "shutdown" }); }; postMessage({ type: "ready" }); @@ -207,8 +234,6 @@ async function setup( const webPoSignalOutput: WebPoSignalOutput = []; const botguardResponse = await botguard.snapshot({ webPoSignalOutput }); - const requestKey = "O43z0dpjhgX20SCx4KAo"; - const integrityTokenResponse = await fetchImpl( buildURL("GenerateIT", true), { @@ -219,13 +244,20 @@ async function setup( "x-user-agent": "grpc-web-javascript/0.1", "user-agent": USER_AGENT, }, - body: JSON.stringify([requestKey, botguardResponse]), + body: JSON.stringify([ + INTEGRITY_TOKEN_REQUEST_KEY, + botguardResponse, + ]), }, ); + if (!integrityTokenResponse.ok) { + throw new Error( + `Could not get integrity token: ${integrityTokenResponse.status}`, + ); + } const integrityTokenBody = IntegrityTokenResponse.parse( await integrityTokenResponse.json(), ); - const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken: integrityTokenBody[0], }, webPoSignalOutput); @@ -238,5 +270,10 @@ async function setup( sessionPoToken, visitorData, generatedMinter: integrityTokenBasedMinter, + cleanup: () => dom.window.close(), }; } + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/lib/potoken/attestation.ts b/src/lib/potoken/attestation.ts new file mode 100644 index 000000000..dc193eda9 --- /dev/null +++ b/src/lib/potoken/attestation.ts @@ -0,0 +1,122 @@ +import type { Innertube } from "youtubei.js"; +import { buildURL, GOOG_API_KEY } from "bgutils"; +import { INTEGRITY_TOKEN_REQUEST_KEY } from "./constants.ts"; + +export type FetchFunction = typeof fetch; + +export interface AttestationChallenge { + program: string; + globalName: string; + interpreterJavascript: string; +} + +export async function getAttestationChallenge( + innertubeClient: Innertube, + fetchImpl: FetchFunction, + userAgent: string, +): Promise { + const response = asRecord( + await innertubeClient.getAttestationChallenge( + "ENGAGEMENT_TYPE_UNBOUND", + ), + ); + const challenge = asRecord(response?.bg_challenge ?? response?.bgChallenge); + if (!challenge) throw new Error("Could not get attestation challenge"); + + const program = getString(challenge, "program"); + const globalName = getString(challenge, "global_name", "globalName"); + if (!program || !globalName) { + throw new Error("Attestation challenge was incomplete"); + } + + const interpreter = asRecord( + challenge.interpreter_javascript ?? challenge.interpreterJavascript, + ); + let interpreterJavascript = interpreter && getString( + interpreter, + "private_do_not_access_or_else_safe_script_wrapped_value", + "privateDoNotAccessOrElseSafeScriptWrappedValue", + ); + if (!interpreterJavascript) { + const interpreterUrl = asRecord( + challenge.interpreter_url ?? challenge.interpreterUrl, + ); + const rawUrl = (interpreterUrl && getString( + interpreterUrl, + "private_do_not_access_or_else_trusted_resource_url_wrapped_value", + "privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", + )) || (interpreter && getString( + interpreter, + "private_do_not_access_or_else_trusted_resource_url_wrapped_value", + "privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", + )); + if (!rawUrl) { + throw new Error("Attestation challenge had no interpreter"); + } + + const scriptResponse = await fetchImpl( + new URL(rawUrl, "https://www.youtube.com/"), + { headers: { accept: "*/*", "user-agent": userAgent } }, + ); + if (!scriptResponse.ok) { + throw new Error( + `Could not load BotGuard interpreter: ${scriptResponse.status}`, + ); + } + interpreterJavascript = await scriptResponse.text(); + } + + if (!interpreterJavascript.trim()) { + throw new Error("BotGuard interpreter was empty"); + } + return { program, globalName, interpreterJavascript }; +} + +export async function getIntegrityToken( + fetchImpl: FetchFunction, + userAgent: string, + botguardResponse: string, +): Promise { + const response = await fetchImpl( + buildURL("GenerateIT", true), + { + method: "POST", + headers: { + "content-type": "application/json+protobuf", + "x-goog-api-key": GOOG_API_KEY, + "x-user-agent": "grpc-web-javascript/0.1", + "user-agent": userAgent, + }, + body: JSON.stringify([ + INTEGRITY_TOKEN_REQUEST_KEY, + botguardResponse, + ]), + }, + ); + if (!response.ok) { + throw new Error(`Could not get integrity token: ${response.status}`); + } + const body = await response.json(); + if (!Array.isArray(body) || typeof body[0] !== "string") { + throw new Error("Integrity token response was invalid"); + } + return body[0]; +} + +function asRecord(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : undefined; +} + +function getString( + record: Record, + ...keys: string[] +): string | undefined { + for (const key of keys) { + if (typeof record[key] === "string" && record[key]) { + return record[key] as string; + } + } + return undefined; +} diff --git a/src/lib/potoken/browserPage.ts b/src/lib/potoken/browserPage.ts new file mode 100644 index 000000000..4c22d6ec0 --- /dev/null +++ b/src/lib/potoken/browserPage.ts @@ -0,0 +1,337 @@ +// The functions below run BotGuard and mint PO tokens *inside the browser +// page* (real Firefox), so they reimplement by hand what bgutils' +// `BG.BotGuardClient` and `BG.WebPoMinter` do in Node — see +// https://github.com/LuanRT/BgUtils. The VM argument order and the short +// error codes (PMD:*, APF:*, ODM:*) mirror that library; revisit both if +// BotGuard changes. The JSDOM fallback (jobs/worker.ts) still uses bgutils +// directly. +import type { Page, Route } from "playwright-core"; +import type { AttestationChallenge } from "./attestation.ts"; + +const PAGE_URL = "https://www.youtube.com/__invidious_companion_potoken__"; +const STATE_PROPERTY = "__invidiousCompanionPoTokenState"; +const OPERATION_TIMEOUT_MS = 10_000; + +export async function initializePoTokenPage(page: Page): Promise { + await page.route("**/*", handlePageRoute); + const response = await page.goto(PAGE_URL, { + waitUntil: "domcontentloaded", + }); + if (!response?.ok()) { + throw new Error( + `Could not initialize browser PO token page: ${response?.status()}`, + ); + } +} + +async function handlePageRoute(route: Route): Promise { + const request = route.request(); + if ( + request.url() === PAGE_URL && request.isNavigationRequest() && + request.resourceType() === "document" + ) { + await route.fulfill({ + status: 200, + contentType: "text/html", + body: + '', + }); + return; + } + await route.abort("blockedbyclient"); +} + +export function runBotGuard( + page: Page, + generationId: string, + challenge: AttestationChallenge, +): Promise { + // Mirrors BG.BotGuardClient and BG.WebPoMinter inside the real browser + // context. Keep this flow aligned with BgUtils when its VM contract changes. + return page.evaluate( + async ({ challenge, generationId, stateProperty, timeoutMs }) => { + type VMFunctions = { + asyncSnapshotFunction: ( + callback: (response: unknown) => void, + args: unknown[], + ) => unknown; + shutdownFunction?: () => unknown; + }; + type GenerationState = { + webPoSignalOutput: unknown[]; + shutdownFunction?: () => unknown; + mintCallback?: (identifier: Uint8Array) => unknown; + }; + type RootState = { + generations: Record; + }; + + const root = globalThis as + & typeof globalThis + & Record; + const state = (root[stateProperty] ??= { + generations: {}, + }) as RootState; + new Function(challenge.interpreterJavascript)(); + const vm = root[challenge.globalName] as + | { a?: (...args: unknown[]) => unknown } + | undefined; + if (!vm?.a) throw new Error("BotGuard VM was unavailable"); + + const webPoSignalOutput: unknown[] = []; + const vmFunctions = await new Promise( + (resolve, reject) => { + let functions: VMFunctions | undefined; + let programLoaded = false; + let settled = false; + const timer = setTimeout( + () => + reject( + new Error("BotGuard initialization timed out"), + ), + timeoutMs, + ); + const finish = () => { + if (!settled && functions && programLoaded) { + settled = true; + clearTimeout(timer); + resolve(functions); + } + }; + const fail = (error: unknown) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(error); + }; + const noOp = () => {}; + try { + const result = vm.a!( + challenge.program, + (snapshot: unknown, shutdown: unknown) => { + if (typeof snapshot !== "function") { + fail( + new Error( + "Snapshot function was unavailable", + ), + ); + return; + } + functions = { + asyncSnapshotFunction: + snapshot as VMFunctions[ + "asyncSnapshotFunction" + ], + shutdownFunction: + typeof shutdown === "function" + ? shutdown as () => unknown + : undefined, + }; + finish(); + }, + true, + undefined, + noOp, + [[], []], + undefined, + false, + undefined, + ); + const synchronousSnapshot = Array.isArray(result) + ? result[0] + : result; + Promise.resolve(synchronousSnapshot).then(() => { + programLoaded = true; + finish(); + }).catch(fail); + } catch (error) { + fail(error); + } + }, + ); + + state.generations[generationId] = { + webPoSignalOutput, + shutdownFunction: vmFunctions.shutdownFunction, + }; + try { + return await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("BotGuard snapshot timed out")), + timeoutMs, + ); + const fail = (error: unknown) => { + clearTimeout(timer); + reject(error); + }; + try { + const result = vmFunctions.asyncSnapshotFunction( + (value) => { + clearTimeout(timer); + if (typeof value === "string" && value) { + resolve(value); + } else { + reject( + new Error( + "BotGuard snapshot was invalid", + ), + ); + } + }, + [ + undefined, + undefined, + webPoSignalOutput, + undefined, + ], + ); + Promise.resolve(result).catch(fail); + } catch (error) { + fail(error); + } + }); + } catch (error) { + delete state.generations[generationId]; + vmFunctions.shutdownFunction?.(); + throw error; + } + }, + { + challenge, + generationId, + stateProperty: STATE_PROPERTY, + timeoutMs: OPERATION_TIMEOUT_MS, + }, + ); +} + +export function createMinter( + page: Page, + generationId: string, + integrityToken: string, +): Promise { + return page.evaluate( + async ({ generationId, integrityToken, stateProperty }) => { + type GenerationState = { + webPoSignalOutput: unknown[]; + mintCallback?: (identifier: Uint8Array) => unknown; + }; + type RootState = { + generations: Record; + }; + const root = globalThis as + & typeof globalThis + & Record; + const state = (root[stateProperty] as RootState | undefined) + ?.generations[generationId]; + if (!state) throw new Error("BotGuard state was unavailable"); + const getMinter = state.webPoSignalOutput[0]; + if (typeof getMinter !== "function") { + throw new Error("PMD:Undefined"); + } + + let normalized = integrityToken + .replace(/-/g, "+") + .replace(/_/g, "/") + .replace(/\./g, "="); + normalized += "=".repeat((4 - normalized.length % 4) % 4); + const bytes = Uint8Array.from( + atob(normalized), + (character) => character.charCodeAt(0), + ); + const mintCallback = await getMinter(bytes); + if (typeof mintCallback !== "function") { + throw new Error("APF:Failed"); + } + state.mintCallback = mintCallback as ( + identifier: Uint8Array, + ) => unknown; + }, + { generationId, integrityToken, stateProperty: STATE_PROPERTY }, + ); +} + +export function mintOnPage( + page: Page, + generationId: string, + contentBinding: string, +): Promise { + return page.evaluate( + async ({ contentBinding, generationId, stateProperty, timeoutMs }) => { + type GenerationState = { + mintCallback?: (identifier: Uint8Array) => unknown; + }; + type RootState = { + generations: Record; + }; + const root = globalThis as + & typeof globalThis + & Record; + const state = (root[stateProperty] as RootState | undefined) + ?.generations[generationId]; + if (typeof state?.mintCallback !== "function") { + throw new Error("PO token minter was unavailable"); + } + + const result = await Promise.race([ + state.mintCallback(new TextEncoder().encode(contentBinding)), + new Promise((_, reject) => + setTimeout( + () => reject(new Error("PO token mint timed out")), + timeoutMs, + ) + ), + ]); + if (!(result instanceof Uint8Array)) throw new Error("ODM:Invalid"); + let binary = ""; + for (const byte of result) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_"); + }, + { + contentBinding, + generationId, + stateProperty: STATE_PROPERTY, + timeoutMs: OPERATION_TIMEOUT_MS, + }, + ); +} + +export function destroyGenerationOnPage( + page: Page, + generationId: string, +): Promise { + return page.evaluate( + async ({ generationId, stateProperty }) => { + type GenerationState = { shutdownFunction?: () => unknown }; + type RootState = { + generations: Record; + }; + const root = globalThis as + & typeof globalThis + & Record; + const generations = (root[stateProperty] as RootState | undefined) + ?.generations; + const state = generations?.[generationId]; + if (!state) return; + delete generations![generationId]; + await Promise.resolve(state.shutdownFunction?.()); + }, + { generationId, stateProperty: STATE_PROPERTY }, + ); +} + +export function destroyAllGenerationsOnPage(page: Page): Promise { + return page.evaluate(async (stateProperty) => { + type GenerationState = { shutdownFunction?: () => unknown }; + type RootState = { generations: Record }; + const root = globalThis as typeof globalThis & Record; + const state = root[stateProperty] as RootState | undefined; + if (!state) return; + delete root[stateProperty]; + await Promise.all( + Object.values(state.generations).map((generation) => + Promise.resolve(generation.shutdownFunction?.()) + ), + ); + }, STATE_PROPERTY); +} diff --git a/src/lib/potoken/constants.ts b/src/lib/potoken/constants.ts new file mode 100644 index 000000000..a942aab00 --- /dev/null +++ b/src/lib/potoken/constants.ts @@ -0,0 +1 @@ +export const INTEGRITY_TOKEN_REQUEST_KEY = "O43z0dpjhgX20SCx4KAo"; diff --git a/src/lib/potoken/errors.ts b/src/lib/potoken/errors.ts new file mode 100644 index 000000000..c6da9fc0f --- /dev/null +++ b/src/lib/potoken/errors.ts @@ -0,0 +1,8 @@ +// Provider-agnostic signal that a browser-backed PO token could not be +// produced (browser missing, could not launch, page failed to initialize). +// The JSDOM fallback keys on this type, so any future browser provider +// (e.g. a CDP-attached browser) should throw it too rather than a +// provider-specific error. +export class BrowserPoTokenUnavailableError extends Error { + override name = "BrowserPoTokenUnavailableError"; +} diff --git a/src/main.ts b/src/main.ts index a6a7eaab9..34da650e1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,7 +1,11 @@ import { Hono } from "hono"; import { companionRoutes, miscRoutes } from "./routes/index.ts"; import { Innertube, Platform } from "youtubei.js"; -import { poTokenGenerate, type TokenMinter } from "./lib/jobs/potoken.ts"; +import { + closePoTokenRuntime, + poTokenGenerate, + type TokenMinter, +} from "./lib/jobs/potoken.ts"; import { USER_AGENT } from "bgutils"; import { retry } from "@std/async"; import type { HonoVariables } from "./lib/types/HonoVariables.ts"; @@ -223,22 +227,27 @@ export function run(signal: AbortSignal, port: number, hostname: string) { ); } } +export async function shutdown() { + await closePoTokenRuntime(); +} + if (import.meta.main) { const controller = new AbortController(); const { signal } = controller; run(signal, config.server.port, config.server.host); - if (Deno.build.os !== "windows") { - Deno.addSignalListener("SIGTERM", () => { - console.log("Caught SIGINT, shutting down..."); - controller.abort(); - Deno.exit(0); - }); - } - - Deno.addSignalListener("SIGINT", () => { - console.log("Caught SIGINT, shutting down..."); + let shuttingDown = false; + const stop = async (signalName: string) => { + if (shuttingDown) return; + shuttingDown = true; + console.log(`Caught ${signalName}, shutting down...`); controller.abort(); + await shutdown(); Deno.exit(0); - }); + }; + + if (Deno.build.os !== "windows") { + Deno.addSignalListener("SIGTERM", () => void stop("SIGTERM")); + } + Deno.addSignalListener("SIGINT", () => void stop("SIGINT")); } diff --git a/src/tests/main_test.ts b/src/tests/main_test.ts index 5d455719c..0fd69bbe6 100644 --- a/src/tests/main_test.ts +++ b/src/tests/main_test.ts @@ -1,5 +1,5 @@ Deno.env.set("SERVER_SECRET_KEY", "aaaaaaaaaaaaaaaa"); -const { run, tokenMinterReady } = await import("../main.ts"); +const { run, shutdown, tokenMinterReady } = await import("../main.ts"); const { parseConfig } = await import("../lib/helpers/config.ts"); const config = await parseConfig(); @@ -16,31 +16,34 @@ Deno.test({ `http://${config.server.host}:${config.server.port.toString()}${config.server.base_path}`; const headers = { Authorization: "Bearer aaaaaaaaaaaaaaaa" }; - await run( - controller.signal, - config.server.port, - config.server.host, - ); - - // Wait for tokenMinter to be ready before running tests - await tokenMinterReady; - - await t.step( - "Check if it can get an OK playabilityStatus on /youtubei/v1/player", - youtubePlayer.bind(null, baseUrl, headers), - ); - - await t.step( - "Check if it can generate a DASH manifest", - dashManifest.bind(null, baseUrl), - ); - - await t.step( - "Check if it can generate a valid URL for latest_version", - latestVersion.bind(null, baseUrl), - ); - - await controller.abort(); + try { + await run( + controller.signal, + config.server.port, + config.server.host, + ); + + // Wait for tokenMinter to be ready before running tests + await tokenMinterReady; + + await t.step( + "Check if it can get an OK playabilityStatus on /youtubei/v1/player", + youtubePlayer.bind(null, baseUrl, headers), + ); + + await t.step( + "Check if it can generate a DASH manifest", + dashManifest.bind(null, baseUrl), + ); + + await t.step( + "Check if it can generate a valid URL for latest_version", + latestVersion.bind(null, baseUrl), + ); + } finally { + controller.abort(); + await shutdown(); + } }, // need to disable leaks test for now because we are leaking resources when using HTTPClient using a proxy sanitizeResources: false, From 9854ec58fbf11ee9cbf7210c8ac8d094d1a13bb3 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 3 Aug 2026 02:43:01 +0200 Subject: [PATCH 2/4] chore: remove ldd hack and just deny ldd bin --- Dockerfile | 1 - deno.jsonc | 6 +++--- docker/ldd | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 docker/ldd diff --git a/Dockerfile b/Dockerfile index ea441a161..da18f794c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -191,7 +191,6 @@ COPY --from=thc-bin /thc /thc COPY --from=tini-bin /tini /tini COPY --from=bootstrap-builder /camoufox-bootstrap /camoufox-bootstrap COPY --from=camoufox-download --chown=10001:10001 /camoufox.zip /usr/share/invidious-companion/camoufox.zip -COPY --chmod=0444 ./docker/ldd /usr/bin/ldd # Copy cache directory and set correct permissions COPY --from=builder --chown=10001:10001 /var/tmp/youtubei.js /var/tmp/youtubei.js diff --git a/deno.jsonc b/deno.jsonc index d05aab413..610080e4c 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -2,11 +2,11 @@ // Deno tasks definitions. "tasks": { // Run the server in development mode with file-watching. - "dev": "DENO_DIR=/var/tmp/youtubei.js/deno-cache XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache deno run --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --watch src/main.ts", + "dev": "DENO_DIR=/var/tmp/youtubei.js/deno-cache XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache deno run --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease,cpus,networkInterfaces --allow-run --allow-ffi --allow-read=.,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup,/usr/bin/ldd --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --watch src/main.ts", // Compile a self-contained executable binary. - "compile": "deno compile --self-extracting --include ./dependencies.json --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --include ./src/lib/jobs/worker.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock src/main.ts --_version_date=\"$(git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g)\" --_version_commit=\"$(git rev-list HEAD --max-count=1 --abbrev-commit)\"", + "compile": "deno compile --self-extracting --include ./dependencies.json --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --include ./src/lib/jobs/worker.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease,cpus,networkInterfaces --allow-run --allow-ffi --allow-read=.,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup,/usr/bin/ldd --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock src/main.ts --_version_date=\"$(git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g)\" --_version_commit=\"$(git rev-list HEAD --max-count=1 --abbrev-commit)\"", // Run integration tests. - "test": "DENO_DIR=/var/tmp/youtubei.js/deno-cache XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache deno test --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease --allow-run --allow-ffi --allow-read=.,/usr/bin/ldd,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock", + "test": "DENO_DIR=/var/tmp/youtubei.js/deno-cache XDG_CACHE_HOME=/var/tmp/youtubei.js/xdg-cache deno test --allow-import=github.com:443,jsr.io:443,cdn.jsdelivr.net:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname,homedir,osRelease,cpus,networkInterfaces --allow-run --allow-ffi --allow-read=.,/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock --deny-read=/proc/version,/etc/os-release,/proc/self/cgroup,/usr/bin/ldd --allow-write=/var/tmp/youtubei.js,/tmp/invidious-companion.sock,/tmp/companionsock", // Check code formatting. "format": "deno fmt --check src/**", // Type-check the source code without emitting output. diff --git a/docker/ldd b/docker/ldd deleted file mode 100644 index 1a5ef72f9..000000000 --- a/docker/ldd +++ /dev/null @@ -1 +0,0 @@ -glibc From db80758d1e48fba04a5367ea04695c4fa2a48af8 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 3 Aug 2026 02:57:03 +0200 Subject: [PATCH 3/4] chore: explain better reason for camoufox-bootstrap.go --- docker/camoufox-bootstrap.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker/camoufox-bootstrap.go b/docker/camoufox-bootstrap.go index 2d87b5804..6b050e09a 100644 --- a/docker/camoufox-bootstrap.go +++ b/docker/camoufox-bootstrap.go @@ -5,6 +5,23 @@ // for the target version (.camoufox.linux[].version = "vX.Y.Z-release"); // keep that parsing in lockstep. (version.json is the generated install // marker, written by install() below.) +// +// Why this runs at container START instead of just unzipping in the Dockerfile: +// CAMOUFOX_INSTALL_DIR (/var/tmp/youtubei.js/camoufox) lives inside the rw +// bind-mounted volume declared in docker-compose.yaml, and the container runs +// with `read_only: true`. So: +// - Unzipping into that path at build time is useless: the runtime bind mount +// shadows the image layer, and bind mounts (unlike named volumes) are never +// seeded from the image, so the path is empty/host-provided at runtime. +// - The volume is the only writable location (read-only rootfs), so the +// browser can only be materialized at runtime, into the volume. +// +// This binary also creates the writable scratch dirs (home/tmp/xdg-cache/ +// xdg-data) the empty volume lacks, and re-extracts on a version.json mismatch. +// Alternative (not chosen): bake the unzipped browser into a non-volume image +// path (e.g. /opt/camoufox, read-only is fine since Firefox writes only to its +// profile) and move the scratch-dir mkdirs into the app — at the cost of a +// larger image and losing the persistent-volume browser cache. package main import ( From 034352817414572eeaf809b9ec7a8f8a32534175 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:16:31 +0200 Subject: [PATCH 4/4] chore: put back user creation --- Dockerfile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index da18f794c..0e39e11b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,13 @@ WORKDIR /src COPY ./docker/camoufox-bootstrap.go ./main.go RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /camoufox-bootstrap ./main.go +# Create the non-privileged appuser in a stage that has useradd; its +# /etc/passwd and /etc/group are copied into the shell-less final image so the +# runtime UID/GID (10001, matching docker-compose `user: 10001:10001`) resolves +# to a real named user instead of an anonymous UID. +FROM dependabot-debian AS user-stage +RUN groupadd --gid 10001 appuser && \ + useradd --uid 10001 --gid 10001 --no-create-home appuser # Stage for downloading files using curl from Debian FROM dependabot-debian AS debian-curl @@ -187,6 +194,10 @@ COPY --from=camoufox-runtime /etc/fonts/ /etc/fonts/ COPY --from=camoufox-runtime /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=camoufox-runtime /var/cache/fontconfig/ /var/cache/fontconfig/ +# Named non-privileged user, so the runtime USER 10001:10001 maps to appuser. +COPY --from=user-stage /etc/passwd /etc/passwd +COPY --from=user-stage /etc/group /etc/group + COPY --from=thc-bin /thc /thc COPY --from=tini-bin /tini /tini COPY --from=bootstrap-builder /camoufox-bootstrap /camoufox-bootstrap @@ -221,7 +232,7 @@ ENV SERVER_BASE_PATH=/companion \ COPY ./config/ ./config/ # Switch to non-privileged user -USER 10001:10001 +USER appuser ENTRYPOINT ["/tini", "--", "/camoufox-bootstrap"] CMD ["/app/invidious_companion"]