Server v3.0.0

.github/actions/setup-r/action.yml

@@ -40,7 +40,22 @@ runs:
     - name: Set up TinyTeX
       uses: r-lib/actions/setup-tinytex@v2
 
-    # libcurl is required for building the R package documentation.
-    - name: Install libcurl
+    # Native libraries required to build the R package and its documentation
+    # toolchain from source. When the R package cache misses, devtools and
+    # pkgdown are compiled from source, which pulls in systemfonts, textshaping
+    # and ragg. Those packages need the font (fontconfig, freetype, harfbuzz,
+    # fribidi) and image (png, tiff, jpeg) development headers, and libcurl is
+    # needed for the documentation build.
+    - name: Install system libraries
       shell: bash
-      run: sudo apt-get install -y libcurl4-openssl-dev
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          libcurl4-openssl-dev \
+          libfontconfig1-dev \
+          libfreetype-dev \
+          libharfbuzz-dev \
+          libfribidi-dev \
+          libpng-dev \
+          libtiff-dev \
+          libjpeg-dev

.github/workflows/server-pre-release.yml

@@ -17,7 +17,7 @@ jobs:
   pre-release-server:
     name: Build and push Docker image
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 105
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -68,12 +68,12 @@ jobs:
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
-          mvn --batch-mode deploy -PdockerPreRelease -Dmaven.deploy.skip \
+          mvn --batch-mode --update-snapshots deploy -PdockerPreRelease -Dmaven.deploy.skip \
           org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \
           -Dsonar.projectKey=aehrc_pathling_server -Dsonar.organization=aehrc \
           -Dsonar.host.url=https://sonarcloud.io \
           -Dsonar.sarifReportPaths=../trivy-results-server.sarif
-        timeout-minutes: 30
+        timeout-minutes: 90
 
       - name: Save test reports
         if: always()

.github/workflows/server-release.yml

@@ -17,7 +17,7 @@ jobs:
     name: Build and push Docker image
     if: github.event_name == 'workflow_dispatch' || startsWith(github.ref_name, 'server-v')
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 105
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -65,7 +65,7 @@ jobs:
           -Dsonar.projectKey=aehrc_pathling_server -Dsonar.organization=aehrc \
           -Dsonar.host.url=https://sonarcloud.io \
           -Dsonar.sarifReportPaths=../trivy-results-server.sarif
-        timeout-minutes: 30
+        timeout-minutes: 90
 
       - name: Save test reports
         if: always()

.github/workflows/server-test.yml

@@ -22,7 +22,7 @@ jobs:
   test-server:
     name: Build and test server
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 45
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -88,7 +88,7 @@ jobs:
           -Dsonar.projectKey=aehrc_pathling_server -Dsonar.organization=aehrc \
           -Dsonar.host.url=https://sonarcloud.io \
           -Dsonar.sarifReportPaths=trivy-results.sarif
-        timeout-minutes: 30
+        timeout-minutes: 45
 
       - name: Save test reports
         if: always()

.github/workflows/ui-test.yml

@@ -1,3 +1,8 @@
+# Runs the Admin UI quality checks documented in ui/CONTRIBUTING.md. Called by
+# both test.yml and pre-release.yml.
+#
+# Author: John Grimes
+
 name: Test UI
 
 on:
@@ -27,11 +32,14 @@ jobs:
       - name: Lint
         run: bun run lint
 
+      - name: Check duplication
+        run: bun run lint:duplication
+
       - name: Build
         run: bun run build
 
-      - name: Run unit tests
-        run: bun run test:run
+      - name: Run unit tests with coverage
+        run: bun run test:coverage
 
       - name: Install Playwright browsers
         # The pinned version must be kept in step with @playwright/test in

CONTRIBUTING.md

@@ -150,16 +150,34 @@ mvn clean install -pl lib/R -am
 
 ### Clearing the Ivy cache
 
-When rebuilding after making changes to upstream modules, you may need to clear
-the local Ivy cache before the changes will be picked up by the Python and R
-libraries. The Ivy cache is typically located at `~/.ivy2/cache`.
-
-To clear the cache:
+When rebuilding after making changes to upstream modules, you need to clear the
+local Ivy state before the changes will be picked up by the Python and R
+libraries.
+
+Spark resolves `--packages` through Ivy, which keeps two directories per Ivy
+version: `cache` (the resolved module metadata and artifacts) and `jars` (the
+jars retrieved for the run). Both must be cleared. Removing only `cache` leaves
+`au.csiro.pathling_library-runtime-<version>-SNAPSHOT.jar` in `jars`, and
+because the SNAPSHOT filename never changes, Ivy reports `0 artifacts copied, N
+already retrieved` and silently reuses the stale build. The tests still run and
+pass; they just do not exercise the new code.
+
+Spark also uses a home directory suffixed with its bundled Ivy version (for
+example `~/.ivy2.5.2`) in addition to the conventional `~/.ivy2`, so both trees
+need to be cleared:
 
 ```bash
-rm -rf ~/.ivy2/cache
+rm -rf ~/.ivy2/cache ~/.ivy2/jars ~/.ivy2.5.2/cache ~/.ivy2.5.2/jars
+mkdir -p ~/.ivy2/cache ~/.ivy2/jars ~/.ivy2.5.2/cache ~/.ivy2.5.2/jars
 ```
 
+Run `ls -d ~/.ivy2*` to confirm which version-suffixed directories exist on your
+machine, as the suffix follows the Spark version in use.
+
+The `mkdir` step matters. With the `jars` directory missing, Ivy fails to
+retrieve `delta-spark`, `delta-storage` and `antlr4-runtime` from the local
+Maven resolver, and the run fails with `JAVA_GATEWAY_EXITED`.
+
 After clearing the cache, rebuild the libraries:
 
 ```bash

benchmark/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>au.csiro.pathling</groupId>
     <artifactId>pathling</artifactId>
-    <version>9.8.0</version>
+    <version>9.9.0-SNAPSHOT</version>
   </parent>
   <artifactId>benchmark</artifactId>
   <packaging>jar</packaging>

deployment/cache/README.md

@@ -8,6 +8,10 @@ based upon the deployment:
 
 - `PATHLING_HOST`: The host name of the Pathling server.
 - `PATHLING_PORT`: The port number exposed by the Pathling server.
+- `PATHLING_FIRST_BYTE_TIMEOUT`: The maximum time to wait for the first byte of
+  a backend response, expressed as a Varnish duration (e.g. `60s`, `10m`). Raise
+  this above the Varnish default of `60s` to accommodate long-running
+  synchronous Pathling queries.
 
-Copyright © 2065, Commonwealth Scientific and Industrial Research Organisation
+Copyright © 2025, Commonwealth Scientific and Industrial Research Organisation
 (CSIRO) ABN 41 687 119 230. Licensed under the Apache License, Version 2.0.

deployment/cache/chart/Chart.yaml

@@ -8,7 +8,7 @@ name: pathling-cache
 description: A Varnish-based frontend cache optimised for use with Pathling Server
 icon: https://raw.githubusercontent.com/aehrc/pathling/main/media/logo-icon-colour-detail.svg
 type: application
-version: 1.0.0
+version: 1.1.0
 maintainers:
   - name: John Grimes
     email: John.Grimes@csiro.au

deployment/cache/chart/README.md

@@ -26,19 +26,20 @@ helm install my-cache ./chart \
 
 ## Configuration
 
-| Parameter                       | Description                                | Default                               |
-| ------------------------------- | ------------------------------------------ | ------------------------------------- |
-| `pathlingCache.image`           | Container image to use                     | `ghcr.io/aehrc/pathling-cache:latest` |
-| `pathlingCache.imagePullPolicy` | Image pull policy                          | `IfNotPresent`                        |
-| `pathlingCache.replicas`        | Number of replicas to deploy               | `1`                                   |
-| `pathlingCache.pathlingHost`    | Hostname of the Pathling server (required) | `~`                                   |
-| `pathlingCache.pathlingPort`    | Port of the Pathling server (required)     | `~`                                   |
-| `pathlingCache.service.type`    | Kubernetes service type                    | `ClusterIP`                           |
-| `pathlingCache.service.port`    | Service port                               | `80`                                  |
-| `pathlingCache.resources`       | CPU/memory resource requests and limits    | `{}`                                  |
-| `pathlingCache.tolerations`     | Pod tolerations                            | `[]`                                  |
-| `pathlingCache.affinity`        | Pod affinity rules                         | `{}`                                  |
-| `pathlingCache.nodeSelector`    | Node selector labels                       | `{}`                                  |
+| Parameter                        | Description                                               | Default                               |
+| -------------------------------- | --------------------------------------------------------- | ------------------------------------- |
+| `pathlingCache.image`            | Container image to use                                    | `ghcr.io/aehrc/pathling-cache:latest` |
+| `pathlingCache.imagePullPolicy`  | Image pull policy                                         | `IfNotPresent`                        |
+| `pathlingCache.replicas`         | Number of replicas to deploy                              | `1`                                   |
+| `pathlingCache.pathlingHost`     | Hostname of the Pathling server (required)                | `~`                                   |
+| `pathlingCache.pathlingPort`     | Port of the Pathling server (required)                    | `~`                                   |
+| `pathlingCache.firstByteTimeout` | Backend first-byte timeout (Varnish duration, e.g. `10m`) | `60s`                                 |
+| `pathlingCache.service.type`     | Kubernetes service type                                   | `ClusterIP`                           |
+| `pathlingCache.service.port`     | Service port                                              | `80`                                  |
+| `pathlingCache.resources`        | CPU/memory resource requests and limits                   | `{}`                                  |
+| `pathlingCache.tolerations`      | Pod tolerations                                           | `[]`                                  |
+| `pathlingCache.affinity`         | Pod affinity rules                                        | `{}`                                  |
+| `pathlingCache.nodeSelector`     | Node selector labels                                      | `{}`                                  |
 
 ## Examples
 

deployment/cache/chart/templates/deployment.yaml

@@ -32,6 +32,8 @@ spec:
               value: {{ required "pathlingCache.pathlingHost is required" .Values.pathlingCache.pathlingHost | quote }}
             - name: PATHLING_PORT
               value: {{ required "pathlingCache.pathlingPort is required" .Values.pathlingCache.pathlingPort | quote }}
+            - name: PATHLING_FIRST_BYTE_TIMEOUT
+              value: {{ .Values.pathlingCache.firstByteTimeout | quote }}
           livenessProbe:
             tcpSocket:
               port: http

deployment/cache/chart/values.schema.json

@@ -33,6 +33,11 @@
           "type": ["integer", "null"],
           "description": "Port of the Pathling server to cache (required)"
         },
+        "firstByteTimeout": {
+          "type": "string",
+          "description": "Maximum time the cache waits for the first byte of a backend response, as a Varnish duration (e.g. '60s', '10m')",
+          "default": "60s"
+        },
         "service": {
           "type": "object",
           "description": "Kubernetes service configuration",

deployment/cache/chart/values.yaml

@@ -12,6 +12,11 @@ pathlingCache:
   pathlingHost: ~
   pathlingPort: ~
 
+  # Maximum time the cache waits for the first byte of a backend response,
+  # expressed as a Varnish duration (e.g. "60s", "10m"). Raise this when
+  # synchronous Pathling queries take longer than the Varnish default of 60s.
+  firstByteTimeout: "60s"
+
   service:
     type: "ClusterIP"
     port: 80

deployment/cache/default.vcl

@@ -7,9 +7,30 @@ import std;
 backend default {
   .host = "${PATHLING_HOST}";
   .port = "${PATHLING_PORT}";
+  // Maximum time to wait for the first byte of the backend response. Raised
+  // above the Varnish default of 60s to accommodate long-running synchronous
+  // Pathling queries.
+  .first_byte_timeout = ${PATHLING_FIRST_BYTE_TIMEOUT};
 }
 
 sub vcl_recv {
+  // Bulk-data file downloads ($result) stream large NDJSON bodies to streaming
+  // clients such as the $import-pnp downloader. The frontend cache must not
+  // gzip, buffer, or cache these: on a cache miss Varnish otherwise serves an
+  // on-the-fly-gzipped, chunked, streamed response that is closed mid-body when
+  // the client reads in bursts (paced by its own downstream writes), truncating
+  // the download. Bypass the cache entirely so the response streams straight
+  // through with the backend's Content-Length.
+  if (req.url ~ "/\$result") {
+    // These clients read in bursts, stalling for as long as it takes to write
+    // each block to their own storage, which can exceed the default one minute
+    // allowed for sending to a client that is not reading. Without this, the
+    // connection is closed part-way through a large file and the download fails
+    // with a premature end of message body.
+    set sess.idle_send_timeout = 30m;
+    set sess.send_timeout = 2h;
+    return (pass);
+  }
   if (req.http.Cache-Control ~ "(private|no-cache|no-store)" || req.http.Pragma == "no-cache") {
     return (pass);
   }
@@ -20,6 +41,16 @@ sub vcl_req_authorization {
 }
 
 sub vcl_backend_response {
+  // Never cache or gzip bulk-data file downloads (see vcl_recv). Disabling gzip
+  // lets the backend Content-Length pass through, so the body is delivered with
+  // a fixed length rather than chunked encoding, which is robust to the bursty,
+  // idle-gap read pattern of streaming download clients.
+  if (bereq.url ~ "/\$result") {
+    set beresp.uncacheable = true;
+    set beresp.do_gzip = false;
+    return (deliver);
+  }
+
   // Respect backend cache-control headers that indicate the response should not be cached.
   if (beresp.http.Cache-Control ~ "(no-cache|no-store|private)") {
     set beresp.uncacheable = true;

deployment/helm/pathling/templates/deployment.yaml

@@ -55,6 +55,14 @@ spec:
               value: "7078"
             - name: spark.driver.bindAddress
               value: 0.0.0.0
+            # In client mode, Spark only sets an ownerReference on executor
+            # pods when it knows the driver pod's name. This allows Kubernetes
+            # garbage collection to remove executor pods (and their on-demand
+            # PVCs) when the driver pod is deleted, e.g. on rollout restart.
+            - name: spark.kubernetes.driver.pod.name
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             {{- range $configKey, $configValue := .Values.pathling.config }}
             - name: {{ $configKey }}
               value: {{ $configValue | quote }}

deployment/helm/pathling/templates/role.yaml

@@ -6,6 +6,11 @@ rules:
   - apiGroups: [""]
     resources: ["pods", "services", "configmaps"]
     verbs: ["create", "get", "list", "watch", "delete"]
+  # Allow the driver to provision and remove the dynamically created scratch
+  # volumes used for executor local storage (claimName OnDemand).
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["create", "get", "list", "watch", "delete"]
   - apiGroups: [""]
     resources: ["pods/log"]
     verbs: ["get", "list", "watch"]

encoders/pom.xml

@@ -32,7 +32,7 @@
   <parent>
     <groupId>au.csiro.pathling</groupId>
     <artifactId>pathling</artifactId>
-    <version>9.8.0</version>
+    <version>9.9.0-SNAPSHOT</version>
   </parent>
   <artifactId>encoders</artifactId>
   <packaging>jar</packaging>