From 6bc3ed992b93f389c7b0e63cb5ab3021080c87e5 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:02:06 +1000 Subject: [PATCH 001/162] feat: Align SQL on FHIR run operation and add cross-cutting rejections Add viewReference support, Reference-typed patient/group with spec cardinality, Bundle unwrapping in the resource parameter, and 422 for semantically invalid ViewDefinitions on the run operation. Across the run, export, and sqlquery-run operations, reject an explicitly unsupported _format and the unsupported source parameter rather than silently ignoring them. Add a creation-time start timestamp to the async job for later export-manifest timing. --- .../java/au/csiro/pathling/async/Job.java | 8 + .../sqlquery/SqlQueryExecutionHelper.java | 16 ++ .../sqlquery/SqlQueryInstanceRunProvider.java | 9 + .../sqlquery/SqlQueryOutputFormat.java | 32 ++- .../sqlquery/SqlQueryRequestParser.java | 4 +- .../sqlquery/SqlQueryRunProvider.java | 4 + .../operations/view/ReferenceParameters.java | 65 +++++ .../view/ViewDefinitionExportProvider.java | 19 +- .../ViewDefinitionInstanceRunProvider.java | 101 ++++--- .../view/ViewDefinitionRunProvider.java | 34 ++- .../operations/view/ViewExecutionHelper.java | 159 ++++++++++- .../operations/view/ViewExportFormat.java | 19 +- .../operations/view/ViewOutputFormat.java | 31 ++- .../java/au/csiro/pathling/async/JobTest.java | 49 ++++ .../sqlquery/SqlQueryOutputFormatTest.java | 43 +++ .../sqlquery/SqlQueryRequestParserTest.java | 29 ++ .../view/ReferenceParametersTest.java | 76 +++++ .../ViewDefinitionExportProviderTest.java | 7 +- .../view/ViewDefinitionRunProviderIT.java | 260 +++++++++++++++++- .../view/ViewDefinitionRunProviderTest.java | 202 +++++++++++++- .../ViewExecutionHelperResolutionTest.java | 159 +++++++++++ .../operations/view/ViewExportFormatTest.java | 23 +- .../operations/view/ViewOutputFormatTest.java | 45 ++- .../SecurityEnabledViewOperationTest.java | 16 +- 24 files changed, 1309 insertions(+), 101 deletions(-) create mode 100644 server/src/main/java/au/csiro/pathling/operations/view/ReferenceParameters.java create mode 100644 server/src/test/java/au/csiro/pathling/async/JobTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/view/ReferenceParametersTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperResolutionTest.java diff --git a/server/src/main/java/au/csiro/pathling/async/Job.java b/server/src/main/java/au/csiro/pathling/async/Job.java index e1db1babea..2c853abcbe 100644 --- a/server/src/main/java/au/csiro/pathling/async/Job.java +++ b/server/src/main/java/au/csiro/pathling/async/Job.java @@ -19,6 +19,7 @@ import jakarta.annotation.Nonnull; import jakarta.servlet.http.HttpServletResponse; +import java.time.Instant; import java.util.Optional; import java.util.concurrent.Future; import java.util.function.Consumer; @@ -56,6 +57,12 @@ public interface JobTag {} /** The identifier of the user who owns this job, if authenticated. */ @Nonnull private final Optional ownerId; + /** + * The time at which this job was created (kick-off time). Used to populate the SQL on FHIR export + * manifest's {@code exportStartTime} and to compute {@code exportDuration}. + */ + @Nonnull private final Instant startTime; + /** The total number of stages in this job, used to calculate progress percentage. */ private int totalStages; @@ -102,6 +109,7 @@ public Job( this.operation = operation; this.result = result; this.ownerId = ownerId; + this.startTime = Instant.now(); this.responseModification = httpServletResponse -> {}; } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java index 2b4d0906e4..3cfbdb41a0 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java @@ -78,6 +78,22 @@ public SqlQueryExecutionHelper( this.libraryReferenceResolver = libraryReferenceResolver; } + /** + * Rejects the unsupported {@code source} parameter (external data source). Pathling does not + * implement external data sources, so a supplied {@code source} value is rejected rather than + * silently ignored. This shared guard backs the {@code source} rejection at the system, type, and + * instance levels, over both POST and GET. + * + * @param source the {@code source} parameter value, if supplied + * @throws InvalidRequestException if {@code source} is present and non-blank + */ + public void rejectSourceParameter(@Nullable final String source) { + if (source != null && !source.isBlank()) { + throw new InvalidRequestException( + "The 'source' parameter (external data source) is not supported by this server."); + } + } + /** * Executes a {@code $sqlquery-run} request and streams results to the HTTP response. Exactly one * of {@code queryResource} and {@code queryReference} must be provided. diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java index a1b16a86df..b9570e3af9 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java @@ -90,6 +90,7 @@ public Class getResourceType() { * @param includeHeader whether to include a header row in CSV output * @param limit the maximum number of rows to return * @param parameters runtime parameter bindings as a Parameters resource + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the servlet request details containing HTTP headers * @param response the HTTP response for streaming output */ @@ -103,9 +104,12 @@ public void runTypeLevel( @Nullable @OperationParam(name = "header") final BooleanType includeHeader, @Nullable @OperationParam(name = "_limit") final IntegerType limit, @Nullable @OperationParam(name = "parameters") final Parameters parameters, + @Nullable @OperationParam(name = "source") final String source, @Nonnull final ServletRequestDetails requestDetails, @Nullable final HttpServletResponse response) { + executionHelper.rejectSourceParameter(source); + final String acceptHeader = requestDetails.getServletRequest().getHeader("Accept"); executionHelper.executeSqlQuery( @@ -128,20 +132,25 @@ public void runTypeLevel( * @param includeHeader whether to include a header row in CSV output * @param limit the maximum number of rows to return * @param parameters runtime parameter bindings as a Parameters resource + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the servlet request details containing HTTP headers * @param response the HTTP response for streaming output */ @Operation(name = "$sqlquery-run", idempotent = true, manualResponse = true) @OperationAccess("sqlquery-run") + @SuppressWarnings("java:S107") public void runById( @IdParam final IdType libraryId, @Nullable @OperationParam(name = "_format") final String format, @Nullable @OperationParam(name = "header") final BooleanType includeHeader, @Nullable @OperationParam(name = "_limit") final IntegerType limit, @Nullable @OperationParam(name = "parameters") final Parameters parameters, + @Nullable @OperationParam(name = "source") final String source, @Nonnull final ServletRequestDetails requestDetails, @Nullable final HttpServletResponse response) { + executionHelper.rejectSourceParameter(source); + final IBaseResource libraryResource = libraryReferenceResolver.resolve(new Reference("Library/" + libraryId.getIdPart())); final String acceptHeader = requestDetails.getServletRequest().getHeader("Accept"); diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java index de87733b3e..cc70fb7ba5 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java @@ -17,6 +17,7 @@ package au.csiro.pathling.operations.sqlquery; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.Arrays; @@ -73,11 +74,38 @@ public static SqlQueryOutputFormat fromString(@Nullable final String format) { if (isNullOrBlank(format)) { return DEFAULT_FORMAT; } + return matchFormat(format).orElse(DEFAULT_FORMAT); + } + + /** + * Parses an explicit {@code _format} parameter value strictly. A null or blank value maps to the + * default (NDJSON); a non-blank value that matches no supported code or media type is rejected. + * + * @param format the explicit {@code _format} value to parse, or null/blank for the default + * @return the corresponding format + * @throws InvalidRequestException if the value is non-blank and not a supported format + */ + @Nonnull + public static SqlQueryOutputFormat fromStringStrict(@Nullable final String format) { + if (isNullOrBlank(format)) { + return DEFAULT_FORMAT; + } + return matchFormat(format) + .orElseThrow( + () -> + new InvalidRequestException( + ("Unsupported _format value '%s'. Supported formats: ndjson, csv, json," + + " parquet, fhir.") + .formatted(format))); + } + + /** Matches a format string against the supported codes and content types. */ + @Nonnull + private static Optional matchFormat(@Nonnull final String format) { final String normalised = format.toLowerCase().trim(); return Arrays.stream(values()) .filter(f -> f.code.equals(normalised) || f.contentType.equals(normalised)) - .findFirst() - .orElse(DEFAULT_FORMAT); + .findFirst(); } /** diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java index 8a566a8d40..6b077fcc0f 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java @@ -135,8 +135,10 @@ private Library castToLibrary(@Nonnull final IBaseResource resource) { @Nonnull private SqlQueryOutputFormat selectOutputFormat( @Nullable final String format, @Nullable final String acceptHeader) { + // An explicit _format parameter is parsed strictly (an unsupported value is rejected), while + // Accept-header negotiation remains lenient and falls back to NDJSON. if (format != null && !format.isBlank()) { - return SqlQueryOutputFormat.fromString(format); + return SqlQueryOutputFormat.fromStringStrict(format); } return SqlQueryOutputFormat.fromAcceptHeader(acceptHeader); } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java index 7f8c09f137..027b7942fe 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java @@ -69,6 +69,7 @@ public SqlQueryRunProvider(@Nonnull final SqlQueryExecutionHelper executionHelpe * @param includeHeader whether to include a header row in CSV output * @param limit the maximum number of rows to return * @param parameters runtime parameter bindings as a Parameters resource + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the servlet request details containing HTTP headers * @param response the HTTP response for streaming output */ @@ -82,9 +83,12 @@ public void run( @Nullable @OperationParam(name = "header") final BooleanType includeHeader, @Nullable @OperationParam(name = "_limit") final IntegerType limit, @Nullable @OperationParam(name = "parameters") final Parameters parameters, + @Nullable @OperationParam(name = "source") final String source, @Nonnull final ServletRequestDetails requestDetails, @Nullable final HttpServletResponse response) { + executionHelper.rejectSourceParameter(source); + final String acceptHeader = requestDetails.getServletRequest().getHeader("Accept"); executionHelper.executeSqlQuery( diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ReferenceParameters.java b/server/src/main/java/au/csiro/pathling/operations/view/ReferenceParameters.java new file mode 100644 index 0000000000..f9989951ca --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/view/ReferenceParameters.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.view; + +import jakarta.annotation.Nullable; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.Reference; + +/** + * Helper for reducing a FHIR {@link Reference} parameter to the logical id of the stored resource + * it points at, for the SQL on FHIR run and export operations. + * + *

Only literal relative references of the form {@code ResourceType/id} (and a bare {@code id}) + * are supported. Canonical, absolute, contained, and logical references are out of scope and are + * treated as unresolvable. + * + * @author John Grimes + */ +final class ReferenceParameters { + + private ReferenceParameters() { + // Utility class; not intended to be instantiated. + } + + /** + * Extracts the logical id from a literal relative {@link Reference}. + * + * @param reference the reference to reduce, may be {@code null} + * @return the logical id (e.g. {@code 123} from {@code Patient/123}, or {@code abc} from a bare + * {@code abc}), or {@code null} when the reference is empty, absolute, or otherwise not a + * resolvable relative reference + */ + @Nullable + static String extractId(@Nullable final Reference reference) { + if (reference == null) { + return null; + } + final String literal = reference.getReference(); + if (literal == null || literal.isBlank()) { + return null; + } + final IIdType element = reference.getReferenceElement(); + // Absolute and canonical URLs are out of scope; only relative references resolve. + if (element.isAbsolute()) { + return null; + } + final String idPart = element.getIdPart(); + return (idPart == null || idPart.isBlank()) ? null : idPart; + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java index 7a00b95f8a..92d025255c 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java @@ -144,6 +144,7 @@ public ViewDefinitionExportProvider( * @param patientIds patient IDs to filter by * @param groupIds group IDs to filter by * @param since filter resources modified after this timestamp + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the request details * @return the parameters result containing the export manifest, or null if cancelled */ @@ -161,6 +162,7 @@ public Parameters export( @Nullable @OperationParam(name = "patient") final List patientIds, @Nullable @OperationParam(name = "group") final List groupIds, @Nullable @OperationParam(name = "_since") final InstantType since, + @Nullable @OperationParam(name = "source") final String source, @Nonnull final ServletRequestDetails requestDetails) { final Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); @@ -185,7 +187,8 @@ public Parameters export( includeHeader, patientIds, groupIds, - since + since, + source }); final String operationCacheKey = computeCacheKeyComponent( @@ -245,6 +248,17 @@ public PreAsyncValidationResult preAsyncValidate( @Nonnull final ServletRequestDetails servletRequestDetails, @Nonnull final Object[] params) throws InvalidRequestException { + // Reject the unsupported source parameter synchronously at kick-off, before any other work. + final String source = (String) params[8]; + if (source != null && !source.isBlank()) { + throw new InvalidRequestException( + "The 'source' parameter (external data source) is not supported by this server."); + } + + // Parse the explicit _format strictly, so an unsupported value is rejected at kick-off + // regardless of the view parameters. + final ViewExportFormat exportFormat = ViewExportFormat.fromString((String) params[3]); + // Extract view parameters from the raw Parameters resource, since HAPI's automatic extraction // does not handle nested part arrays containing resources correctly. final List views = extractViewInputsFromRequest(servletRequestDetails); @@ -264,7 +278,6 @@ public PreAsyncValidationResult preAsyncValidate( // Other parameters are extracted correctly by HAPI. final String clientTrackingId = (String) params[2]; - final String format = (String) params[3]; final BooleanType includeHeader = (BooleanType) params[4]; @SuppressWarnings("unchecked") @@ -289,7 +302,7 @@ public PreAsyncValidationResult preAsyncValidate( servletRequestDetails.getFhirServerBase(), views, clientTrackingId, - ViewExportFormat.fromString(format), + exportFormat, header, allPatientIds, since); diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionInstanceRunProvider.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionInstanceRunProvider.java index ee218739ab..44b13e7c30 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionInstanceRunProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionInstanceRunProvider.java @@ -18,14 +18,12 @@ package au.csiro.pathling.operations.view; import au.csiro.pathling.encoders.ViewDefinitionResource; -import au.csiro.pathling.errors.ResourceNotFoundError; -import au.csiro.pathling.read.ReadExecutor; import au.csiro.pathling.security.OperationAccess; import ca.uhn.fhir.rest.annotation.IdParam; import ca.uhn.fhir.rest.annotation.Operation; import ca.uhn.fhir.rest.annotation.OperationParam; import ca.uhn.fhir.rest.server.IResourceProvider; -import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.servlet.http.HttpServletResponse; @@ -35,6 +33,7 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.InstantType; import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -44,7 +43,8 @@ *

This provides operations at: * *

    - *
  • {@code POST /fhir/ViewDefinition/$run} - Type-level operation with viewResource parameter + *
  • {@code POST /fhir/ViewDefinition/$run} - Type-level operation with a viewResource or + * viewReference parameter *
  • {@code GET /fhir/ViewDefinition/[id]/$run} - Instance-level operation using stored * ViewDefinition *
  • {@code POST /fhir/ViewDefinition/[id]/$run} - Instance-level operation with additional @@ -61,20 +61,15 @@ public class ViewDefinitionInstanceRunProvider implements IResourceProvider { @Nonnull private final ViewExecutionHelper viewExecutionHelper; - @Nonnull private final ReadExecutor readExecutor; - /** * Constructs a new ViewDefinitionInstanceRunProvider. * - * @param viewExecutionHelper the helper for executing view queries - * @param readExecutor the read executor for reading stored ViewDefinitions + * @param viewExecutionHelper the helper for executing view queries and resolving stored + * ViewDefinitions */ @Autowired - public ViewDefinitionInstanceRunProvider( - @Nonnull final ViewExecutionHelper viewExecutionHelper, - @Nonnull final ReadExecutor readExecutor) { + public ViewDefinitionInstanceRunProvider(@Nonnull final ViewExecutionHelper viewExecutionHelper) { this.viewExecutionHelper = viewExecutionHelper; - this.readExecutor = readExecutor; } @Override @@ -83,16 +78,20 @@ public Class getResourceType() { } /** - * Type-level $run operation that accepts a ViewDefinition inline. + * Type-level $run operation that accepts a ViewDefinition inline ({@code viewResource}) or by + * reference ({@code viewReference}). * - * @param viewResource the ViewDefinition resource - * @param format the output format (ndjson or csv), overrides Accept header if provided + * @param viewResource the inline ViewDefinition resource (mutually exclusive with viewReference) + * @param viewReference a reference to a stored ViewDefinition (mutually exclusive with + * viewResource) + * @param format the output format (ndjson, csv, or json), overrides Accept header if provided * @param includeHeader whether to include a header row in CSV output * @param limit the maximum number of rows to return - * @param patientIds patient IDs to filter by - * @param groupIds group IDs to filter by + * @param patient a single patient reference to filter by + * @param group group references to filter by * @param since filter by meta.lastUpdated >= value * @param inlineResources FHIR resources to use instead of the main data source + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the servlet request details containing HTTP headers * @param response the HTTP response for streaming output */ @@ -100,21 +99,30 @@ public Class getResourceType() { @Operation(name = "$run", idempotent = true, manualResponse = true) @OperationAccess("view-run") public void runTypeLevel( - @Nonnull @OperationParam(name = "viewResource") final IBaseResource viewResource, + @Nullable @OperationParam(name = "viewResource") final IBaseResource viewResource, + @Nullable @OperationParam(name = "viewReference") final Reference viewReference, @Nullable @OperationParam(name = "_format") final String format, @Nullable @OperationParam(name = "header") final BooleanType includeHeader, @Nullable @OperationParam(name = "_limit") final IntegerType limit, - @Nullable @OperationParam(name = "patient") final List patientIds, - @Nullable @OperationParam(name = "group") final List groupIds, + @Nullable @OperationParam(name = "patient", max = OperationParam.MAX_UNLIMITED) + final List patient, + @Nullable @OperationParam(name = "group", max = OperationParam.MAX_UNLIMITED) + final List group, @Nullable @OperationParam(name = "_since") final InstantType since, @Nullable @OperationParam(name = "resource") final List inlineResources, - @Nonnull final ca.uhn.fhir.rest.server.servlet.ServletRequestDetails requestDetails, + @Nullable @OperationParam(name = "source") final String source, + @Nonnull final ServletRequestDetails requestDetails, @Nullable final HttpServletResponse response) { + viewExecutionHelper.rejectSourceParameter(source); + + final IBaseResource view = viewExecutionHelper.resolveViewInput(viewResource, viewReference); + final List patientIds = viewExecutionHelper.toPatientIds(patient); + final List groupIds = viewExecutionHelper.toGroupIds(group); final String acceptHeader = requestDetails.getServletRequest().getHeader("Accept"); viewExecutionHelper.executeView( - viewResource, + view, format, acceptHeader, includeHeader, @@ -127,17 +135,19 @@ public void runTypeLevel( } /** - * Instance-level $run operation that uses a stored ViewDefinition by ID. Supports both GET and - * POST requests. + * Instance-level $run operation that uses a stored ViewDefinition by ID. The view is inferred + * from the path; a {@code viewReference} parameter is neither required nor consulted. Supports + * both GET and POST requests. * * @param viewDefinitionId the ID of the stored ViewDefinition - * @param format the output format (ndjson or csv), overrides Accept header if provided + * @param format the output format (ndjson, csv, or json), overrides Accept header if provided * @param includeHeader whether to include a header row in CSV output * @param limit the maximum number of rows to return - * @param patientIds patient IDs to filter by - * @param groupIds group IDs to filter by + * @param patient a single patient reference to filter by + * @param group group references to filter by * @param since filter by meta.lastUpdated >= value * @param inlineResources FHIR resources to use instead of the main data source + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the servlet request details containing HTTP headers * @param response the HTTP response for streaming output */ @@ -149,20 +159,27 @@ public void runById( @Nullable @OperationParam(name = "_format") final String format, @Nullable @OperationParam(name = "header") final BooleanType includeHeader, @Nullable @OperationParam(name = "_limit") final IntegerType limit, - @Nullable @OperationParam(name = "patient") final List patientIds, - @Nullable @OperationParam(name = "group") final List groupIds, + @Nullable @OperationParam(name = "patient", max = OperationParam.MAX_UNLIMITED) + final List patient, + @Nullable @OperationParam(name = "group", max = OperationParam.MAX_UNLIMITED) + final List group, @Nullable @OperationParam(name = "_since") final InstantType since, @Nullable @OperationParam(name = "resource") final List inlineResources, - @Nonnull final ca.uhn.fhir.rest.server.servlet.ServletRequestDetails requestDetails, + @Nullable @OperationParam(name = "source") final String source, + @Nonnull final ServletRequestDetails requestDetails, @Nullable final HttpServletResponse response) { - // Read the ViewDefinition by ID. - final IBaseResource viewResource = readViewDefinition(viewDefinitionId); + viewExecutionHelper.rejectSourceParameter(source); + // The view is inferred from the path id; viewReference is not consulted at the instance level. + final IBaseResource view = + viewExecutionHelper.readStoredViewDefinition(viewDefinitionId.getIdPart()); + final List patientIds = viewExecutionHelper.toPatientIds(patient); + final List groupIds = viewExecutionHelper.toGroupIds(group); final String acceptHeader = requestDetails.getServletRequest().getHeader("Accept"); viewExecutionHelper.executeView( - viewResource, + view, format, acceptHeader, includeHeader, @@ -173,22 +190,4 @@ public void runById( inlineResources, response); } - - /** Reads a ViewDefinition resource by ID. */ - @Nonnull - private IBaseResource readViewDefinition(@Nonnull final IdType viewDefinitionId) { - try { - return readExecutor.read("ViewDefinition", viewDefinitionId.getIdPart()); - } catch (final ResourceNotFoundError e) { - throw new ResourceNotFoundException( - "ViewDefinition with ID '" + viewDefinitionId.getIdPart() + "' not found"); - } catch (final IllegalArgumentException e) { - // Handle case where no ViewDefinition data exists in the data source. - if (e.getMessage() != null && e.getMessage().contains("No data found for resource type")) { - throw new ResourceNotFoundException( - "ViewDefinition with ID '" + viewDefinitionId.getIdPart() + "' not found"); - } - throw e; - } - } } diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionRunProvider.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionRunProvider.java index c59a4d25ee..9a1d2b43c0 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionRunProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionRunProvider.java @@ -30,6 +30,7 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.InstantType; import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -60,17 +61,21 @@ public ViewDefinitionRunProvider(@Nonnull final ViewExecutionHelper viewExecutio } /** - * Executes a ViewDefinition provided inline and returns the results in NDJSON or CSV format with - * chunked streaming. This is the system-level operation at /fhir/$viewdefinition-run. + * Executes a ViewDefinition supplied inline ({@code viewResource}) or by reference ({@code + * viewReference}) and returns the results in the negotiated format with chunked streaming. This + * is the system-level operation at /fhir/$viewdefinition-run. * - * @param viewResource the ViewDefinition resource - * @param format the output format (ndjson or csv), overrides Accept header if provided + * @param viewResource the inline ViewDefinition resource (mutually exclusive with viewReference) + * @param viewReference a reference to a stored ViewDefinition (mutually exclusive with + * viewResource) + * @param format the output format (ndjson, csv, or json), overrides Accept header if provided * @param includeHeader whether to include a header row in CSV output * @param limit the maximum number of rows to return - * @param patientIds patient IDs to filter by - * @param groupIds group IDs to filter by + * @param patient a single patient reference to filter by + * @param group group references to filter by * @param since filter by meta.lastUpdated >= value * @param inlineResources FHIR resources to use instead of the main data source + * @param source the unsupported external data source parameter, rejected when supplied * @param requestDetails the servlet request details containing HTTP headers * @param response the HTTP response for streaming output */ @@ -78,21 +83,30 @@ public ViewDefinitionRunProvider(@Nonnull final ViewExecutionHelper viewExecutio @Operation(name = "$viewdefinition-run", idempotent = true, manualResponse = true) @OperationAccess("view-run") public void run( - @Nonnull @OperationParam(name = "viewResource") final IBaseResource viewResource, + @Nullable @OperationParam(name = "viewResource") final IBaseResource viewResource, + @Nullable @OperationParam(name = "viewReference") final Reference viewReference, @Nullable @OperationParam(name = "_format") final String format, @Nullable @OperationParam(name = "header") final BooleanType includeHeader, @Nullable @OperationParam(name = "_limit") final IntegerType limit, - @Nullable @OperationParam(name = "patient") final List patientIds, - @Nullable @OperationParam(name = "group") final List groupIds, + @Nullable @OperationParam(name = "patient", max = OperationParam.MAX_UNLIMITED) + final List patient, + @Nullable @OperationParam(name = "group", max = OperationParam.MAX_UNLIMITED) + final List group, @Nullable @OperationParam(name = "_since") final InstantType since, @Nullable @OperationParam(name = "resource") final List inlineResources, + @Nullable @OperationParam(name = "source") final String source, @Nonnull final ServletRequestDetails requestDetails, @Nullable final HttpServletResponse response) { + viewExecutionHelper.rejectSourceParameter(source); + + final IBaseResource view = viewExecutionHelper.resolveViewInput(viewResource, viewReference); + final List patientIds = viewExecutionHelper.toPatientIds(patient); + final List groupIds = viewExecutionHelper.toGroupIds(group); final String acceptHeader = requestDetails.getServletRequest().getHeader("Accept"); viewExecutionHelper.executeView( - viewResource, + view, format, acceptHeader, includeHeader, diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java index 50165074be..1c798858c8 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java @@ -19,11 +19,13 @@ import au.csiro.pathling.config.ServerConfiguration; import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.errors.ResourceNotFoundError; import au.csiro.pathling.errors.UnsupportedFhirPathFeatureError; import au.csiro.pathling.io.source.DataSource; import au.csiro.pathling.library.io.source.QueryableDataSource; import au.csiro.pathling.operations.compartment.GroupMemberService; import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import au.csiro.pathling.read.ReadExecutor; import au.csiro.pathling.security.PathlingAuthority; import au.csiro.pathling.security.ResourceAccess.AccessType; import au.csiro.pathling.security.SecurityAspect; @@ -34,6 +36,8 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import jakarta.annotation.Nonnull; @@ -55,9 +59,11 @@ import org.apache.spark.sql.types.StructType; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.InstantType; import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -89,6 +95,8 @@ public class ViewExecutionHelper { @Nonnull private final ResultStreamingHelper streamingHelper; + @Nonnull private final ReadExecutor readExecutor; + /** * Constructs a new ViewExecutionHelper. * @@ -99,7 +107,9 @@ public class ViewExecutionHelper { * @param patientCompartmentService the patient compartment service * @param groupMemberService the group member service * @param serverConfiguration the server configuration + * @param readExecutor the read executor for resolving stored ViewDefinitions by id */ + @SuppressWarnings("java:S107") @Autowired public ViewExecutionHelper( @Nonnull final SparkSession sparkSession, @@ -108,7 +118,8 @@ public ViewExecutionHelper( @Nonnull final FhirEncoders fhirEncoders, @Nonnull final PatientCompartmentService patientCompartmentService, @Nonnull final GroupMemberService groupMemberService, - @Nonnull final ServerConfiguration serverConfiguration) { + @Nonnull final ServerConfiguration serverConfiguration, + @Nonnull final ReadExecutor readExecutor) { this.sparkSession = sparkSession; this.deltaLake = deltaLake; this.fhirContext = fhirContext; @@ -116,10 +127,127 @@ public ViewExecutionHelper( this.patientCompartmentService = patientCompartmentService; this.groupMemberService = groupMemberService; this.serverConfiguration = serverConfiguration; + this.readExecutor = readExecutor; this.gson = ViewDefinitionGson.create(); this.streamingHelper = new ResultStreamingHelper(gson); } + /** + * Rejects the unsupported {@code source} parameter (external data source). Pathling does not + * implement external data sources, so a supplied {@code source} value is rejected rather than + * silently ignored, which would mislead the client about the data that was queried. + * + * @param source the {@code source} parameter value, if supplied + * @throws InvalidRequestException if {@code source} is present and non-blank + */ + public void rejectSourceParameter(@Nullable final String source) { + if (source != null && !source.isBlank()) { + throw new InvalidRequestException( + "The 'source' parameter (external data source) is not supported by this server."); + } + } + + /** + * Resolves the ViewDefinition to run from the mutually exclusive {@code viewResource} and {@code + * viewReference} parameters, for the system and type levels. Exactly one must be supplied. + * + * @param viewResource the inline ViewDefinition resource, if supplied + * @param viewReference a literal relative reference to a stored ViewDefinition, if supplied + * @return the ViewDefinition resource to execute + * @throws InvalidRequestException (400) if both or neither parameter is supplied + * @throws ResourceNotFoundException (404) if the reference does not resolve to a stored + * ViewDefinition + */ + @Nonnull + public IBaseResource resolveViewInput( + @Nullable final IBaseResource viewResource, @Nullable final Reference viewReference) { + final boolean hasResource = viewResource != null; + final boolean hasReference = viewReference != null && !viewReference.isEmpty(); + + if (hasResource && hasReference) { + throw new InvalidRequestException( + "Provide exactly one of 'viewResource' or 'viewReference', not both."); + } + if (!hasResource && !hasReference) { + throw new InvalidRequestException( + "Either 'viewResource' or 'viewReference' must be provided."); + } + if (hasResource) { + return viewResource; + } + + final String id = ReferenceParameters.extractId(viewReference); + if (id == null) { + throw new ResourceNotFoundException( + "The viewReference does not resolve to a stored ViewDefinition."); + } + return readStoredViewDefinition(id); + } + + /** + * Reads a stored ViewDefinition by its logical id, mapping a missing resource to a 404. + * + * @param id the logical id of the stored ViewDefinition + * @return the stored ViewDefinition resource + * @throws ResourceNotFoundException (404) if no ViewDefinition with the id exists + */ + @Nonnull + public IBaseResource readStoredViewDefinition(@Nonnull final String id) { + try { + return readExecutor.read("ViewDefinition", id); + } catch (final ResourceNotFoundError e) { + throw new ResourceNotFoundException("ViewDefinition with ID '" + id + "' not found"); + } catch (final IllegalArgumentException e) { + // Handle the case where no ViewDefinition data exists in the data source at all. + if (e.getMessage() != null && e.getMessage().contains("No data found for resource type")) { + throw new ResourceNotFoundException("ViewDefinition with ID '" + id + "' not found"); + } + throw e; + } + } + + /** + * Reduces the {@code patient} parameter (a {@code Reference} capped at one) to the logical id + * used for compartment filtering. + * + * @param patients the supplied {@code patient} references, if any + * @return a list containing the single patient logical id, or an empty list + * @throws InvalidRequestException (400) if more than one {@code patient} is supplied + */ + @Nonnull + public List toPatientIds(@Nullable final List patients) { + if (patients == null || patients.isEmpty()) { + return List.of(); + } + if (patients.size() > 1) { + throw new InvalidRequestException("Only a single 'patient' parameter is supported."); + } + final String id = ReferenceParameters.extractId(patients.get(0)); + return id == null ? List.of() : List.of(id); + } + + /** + * Reduces the repeatable {@code group} parameter ({@code Reference} values) to the logical ids + * used for compartment filtering. + * + * @param groups the supplied {@code group} references, if any + * @return the list of group logical ids, in order + */ + @Nonnull + public List toGroupIds(@Nullable final List groups) { + if (groups == null || groups.isEmpty()) { + return List.of(); + } + final List ids = new ArrayList<>(); + for (final Reference group : groups) { + final String id = ReferenceParameters.extractId(group); + if (id != null) { + ids.add(new IdType(id)); + } + } + return ids; + } + /** * Executes a ViewDefinition and streams the results to the HTTP response. * @@ -160,10 +288,11 @@ public void executeView( PathlingAuthority.resourceAccess(AccessType.READ, view.getResource())); } - // Determine output format: _format parameter overrides Accept header. + // Determine output format: an explicit _format parameter is parsed strictly (an unsupported + // value is rejected), while Accept-header negotiation remains lenient and falls back to NDJSON. final ViewOutputFormat outputFormat = (format != null && !format.isBlank()) - ? ViewOutputFormat.fromString(format) + ? ViewOutputFormat.fromStringStrict(format) : ViewOutputFormat.fromAcceptHeader(acceptHeader); final boolean shouldIncludeHeader = includeHeader == null || includeHeader.booleanValue(); @@ -212,7 +341,9 @@ private void executeAndStreamResults( try { result = executor.buildQuery(view); } catch (final ConstraintViolationException e) { - throw new InvalidRequestException("Invalid ViewDefinition: " + e.getMessage()); + // A well-formed request carrying a semantically invalid ViewDefinition maps to 422, + // distinct from the 400 used for malformed requests and invalid parameters. + throw new UnprocessableEntityException("Invalid ViewDefinition: " + e.getMessage()); } catch (final UnsupportedOperationException | UnsupportedFhirPathFeatureError e) { // Thrown when a FHIRPath expression is not supported, such as accessing a choice element // without specifying the type via ofType(). @@ -301,17 +432,33 @@ private DataSource buildDataSource( return dataSource; } - /** Parses inline FHIR resources from JSON strings. */ + /** + * Parses inline FHIR resources from JSON strings, unwrapping any {@code Bundle} value into its + * entry resources. A {@code Bundle} contributes its {@code entry[*].resource} members (one level + * of unwrapping); standalone resources are used directly. The data source is the union of all + * standalone resources and all unwrapped Bundle entry resources; an empty Bundle contributes + * nothing. + */ @Nonnull private List parseInlineResources(@Nonnull final List inlineResources) { final IParser jsonParser = fhirContext.newJsonParser(); final List resources = new ArrayList<>(); for (final String resourceJson : inlineResources) { + final IBaseResource parsed; try { - resources.add(jsonParser.parseResource(resourceJson)); + parsed = jsonParser.parseResource(resourceJson); } catch (final Exception e) { throw new InvalidRequestException("Invalid inline resource: " + e.getMessage()); } + if (parsed instanceof final Bundle bundle) { + for (final Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.hasResource()) { + resources.add(entry.getResource()); + } + } + } else { + resources.add(parsed); + } } return resources; } diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java index 09aaea4338..d2a50fa73f 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java @@ -17,6 +17,7 @@ package au.csiro.pathling.operations.view; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.Arrays; @@ -55,10 +56,16 @@ public enum ViewExportFormat { } /** - * Parses a format string into a ViewExportFormat. + * Parses an explicit {@code _format} parameter value into a ViewExportFormat. A null or blank + * value maps to the default (NDJSON); a non-blank value that matches no supported code or media + * type is rejected. * - * @param format the format string to parse, or null for default - * @return the corresponding ViewExportFormat, defaulting to NDJSON + *

    The export operation has a single explicit {@code _format} entry point and no lenient + * content-negotiation path, so this parser is strict. + * + * @param format the explicit {@code _format} value to parse, or null/blank for the default + * @return the corresponding ViewExportFormat + * @throws InvalidRequestException if the value is non-blank and not a supported format */ @Nonnull public static ViewExportFormat fromString(@Nullable final String format) { @@ -69,6 +76,10 @@ public static ViewExportFormat fromString(@Nullable final String format) { return Arrays.stream(values()) .filter(f -> f.code.equals(normalised) || f.contentType.equals(normalised)) .findFirst() - .orElse(NDJSON); + .orElseThrow( + () -> + new InvalidRequestException( + "Unsupported _format value '%s'. Supported formats: ndjson, csv, parquet." + .formatted(format))); } } diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java index 18d4a32fac..ab633d96bc 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java @@ -17,6 +17,7 @@ package au.csiro.pathling.operations.view; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.Arrays; @@ -64,11 +65,37 @@ public static ViewOutputFormat fromString(@Nullable final String format) { if (isNullOrBlank(format)) { return DEFAULT_FORMAT; } + return matchFormat(format).orElse(DEFAULT_FORMAT); + } + + /** + * Parses an explicit {@code _format} parameter value strictly. A null or blank value maps to the + * default (NDJSON); a non-blank value that matches no supported code or media type is rejected. + * + * @param format the explicit {@code _format} value to parse, or null/blank for the default + * @return the corresponding ViewOutputFormat + * @throws InvalidRequestException if the value is non-blank and not a supported format + */ + @Nonnull + public static ViewOutputFormat fromStringStrict(@Nullable final String format) { + if (isNullOrBlank(format)) { + return DEFAULT_FORMAT; + } + return matchFormat(format) + .orElseThrow( + () -> + new InvalidRequestException( + "Unsupported _format value '%s'. Supported formats: ndjson, csv, json." + .formatted(format))); + } + + /** Matches a format string against the supported codes and content types. */ + @Nonnull + private static Optional matchFormat(@Nonnull final String format) { final String normalised = format.toLowerCase().trim(); return Arrays.stream(values()) .filter(f -> f.code.equals(normalised) || f.contentType.equals(normalised)) - .findFirst() - .orElse(DEFAULT_FORMAT); + .findFirst(); } /** diff --git a/server/src/test/java/au/csiro/pathling/async/JobTest.java b/server/src/test/java/au/csiro/pathling/async/JobTest.java new file mode 100644 index 0000000000..b5b5bf01fa --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/async/JobTest.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.async; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Parameters; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link Job}, covering the creation-time start timestamp used by the SQL on FHIR + * export manifest's timing fields. + * + * @author John Grimes + */ +class JobTest { + + @Test + void recordsNonNullStartTimeAtCreation() { + final Instant before = Instant.now(); + final Future result = CompletableFuture.completedFuture(new Parameters()); + final Job job = new Job<>("job-1", "viewdefinition-export", result, Optional.empty()); + final Instant after = Instant.now(); + + // The job records a creation timestamp that falls within the window in which it was created. + assertThat(job.getStartTime()).isNotNull(); + assertThat(job.getStartTime()).isBetween(before, after); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java index 25b1f41fc0..5faa72dd2e 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java @@ -18,7 +18,9 @@ package au.csiro.pathling.operations.sqlquery; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -128,6 +130,47 @@ void fromAcceptHeaderHandlesParametersOtherThanQ() { .isEqualTo(SqlQueryOutputFormat.CSV); } + // --------------------------------------------------------------------------- + // fromStringStrict parsing tests (used for the explicit _format parameter) + // --------------------------------------------------------------------------- + + @ParameterizedTest(name = "fromStringStrict(\"{0}\") returns {1}") + @CsvSource({ + "ndjson, NDJSON", + "csv, CSV", + "json, JSON", + "parquet, PARQUET", + "fhir, FHIR", + "application/x-ndjson, NDJSON", + "text/csv, CSV", + "application/json, JSON", + "application/vnd.apache.parquet, PARQUET", + "application/fhir+json, FHIR" + }) + void fromStringStrictAcceptsEverySupportedCodeAndMediaType( + final String input, final SqlQueryOutputFormat expected) { + assertThat(SqlQueryOutputFormat.fromStringStrict(input)).isEqualTo(expected); + } + + @Test + void fromStringStrictDefaultsToNdjsonForNull() { + assertThat(SqlQueryOutputFormat.fromStringStrict(null)).isEqualTo(SqlQueryOutputFormat.NDJSON); + } + + @ParameterizedTest + @ValueSource(strings = {"", " "}) + void fromStringStrictDefaultsToNdjsonForBlank(final String input) { + assertThat(SqlQueryOutputFormat.fromStringStrict(input)).isEqualTo(SqlQueryOutputFormat.NDJSON); + } + + @Test + void fromStringStrictRejectsUnknownNamingValue() { + // An explicit unsupported format is rejected with the unsupported value named. + assertThatThrownBy(() -> SqlQueryOutputFormat.fromStringStrict("xml")) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("xml"); + } + @Test void getCodeAndContentTypeAreExposed() { assertThat(SqlQueryOutputFormat.NDJSON.getCode()).isEqualTo("ndjson"); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java index 6f0eb1700b..a318fb546c 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java @@ -274,6 +274,35 @@ void rejectsRuntimeParameterSuppliedMoreThanOnce() { .hasMessageContaining("more than once"); } + // --------------------------------------------------------------------------- + // Output-format selection: strict for explicit _format, lenient for Accept. + // --------------------------------------------------------------------------- + + @Test + void rejectsUnsupportedExplicitFormatNamingValue() { + // An explicit _format that is not supported is rejected with the value named. + final Library library = libraryWithSql("SELECT 1"); + assertThatThrownBy(() -> parser.parse(library, "xml", null, null, null, null)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("xml"); + } + + @Test + void honoursSupportedExplicitFormat() { + final Library library = libraryWithSql("SELECT 1"); + final SqlQueryRequest request = parser.parse(library, "csv", null, null, null, null); + assertThat(request.getOutputFormat()).isEqualTo(SqlQueryOutputFormat.CSV); + } + + @Test + void fallsBackToNdjsonWhenAcceptHeaderDoesNotMatch() { + // An Accept header that matches no supported media type is not rejected; it defaults to NDJSON. + final Library library = libraryWithSql("SELECT 1"); + final SqlQueryRequest request = + parser.parse(library, null, "application/xml", null, null, null); + assertThat(request.getOutputFormat()).isEqualTo(SqlQueryOutputFormat.NDJSON); + } + /** Builds a minimal SQLQuery-typed Library carrying the given SQL. */ private static Library libraryWithSql(final String sql) { final Library library = new Library(); diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ReferenceParametersTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ReferenceParametersTest.java new file mode 100644 index 0000000000..689ea0989e --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/view/ReferenceParametersTest.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.view; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hl7.fhir.r4.model.Reference; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link ReferenceParameters}, the logical-id extractor shared by the run and export + * operations' reference parameters. + * + * @author John Grimes + */ +class ReferenceParametersTest { + + @Test + void extractsIdFromTypedRelativeReference() { + // A literal relative reference of the form ResourceType/id yields its id part. + assertThat(ReferenceParameters.extractId(new Reference("Patient/123"))).isEqualTo("123"); + } + + @Test + void extractsIdFromViewDefinitionReference() { + assertThat(ReferenceParameters.extractId(new Reference("ViewDefinition/abc"))).isEqualTo("abc"); + } + + @Test + void extractsIdFromBareIdReference() { + // A bare id (no resource type prefix) is itself the logical id. + assertThat(ReferenceParameters.extractId(new Reference("abc"))).isEqualTo("abc"); + } + + @Test + void returnsNullForNullReference() { + assertThat(ReferenceParameters.extractId(null)).isNull(); + } + + @Test + void returnsNullForEmptyReference() { + // A reference carrying no literal reference string is not resolvable. + assertThat(ReferenceParameters.extractId(new Reference())).isNull(); + } + + @Test + void returnsNullForReferenceWithOnlyDisplay() { + final Reference reference = new Reference(); + reference.setDisplay("A view with no literal reference"); + assertThat(ReferenceParameters.extractId(reference)).isNull(); + } + + @Test + void returnsNullForAbsoluteUrlReference() { + // Absolute URLs are out of scope for resolution and are treated as unresolvable. + assertThat( + ReferenceParameters.extractId( + new Reference("http://example.com/fhir/ViewDefinition/abc"))) + .isNull(); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java index e9a0605949..ee24d7dc59 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java @@ -222,7 +222,7 @@ void exportReturnsResultFromAsyncContext() { AsyncJobContext.setCurrentJob(job); final Parameters result = - provider.export(null, null, null, null, null, null, null, null, requestDetails); + provider.export(null, null, null, null, null, null, null, null, null, requestDetails); assertNotNull(result); verify(exportResultRegistry).put(eq("job-1"), any(ExportResult.class)); @@ -251,7 +251,7 @@ void exportReturnsNullWhenCancelled() { AsyncJobContext.setCurrentJob(job); final Parameters result = - provider.export(null, null, null, null, null, null, null, null, requestDetails); + provider.export(null, null, null, null, null, null, null, null, null, requestDetails); assertNull(result); } @@ -263,6 +263,7 @@ void exportThrowsWhenNoViewsProvided() { assertThrows( InvalidRequestException.class, - () -> provider.export(null, null, null, null, null, null, null, null, requestDetails)); + () -> + provider.export(null, null, null, null, null, null, null, null, null, requestDetails)); } } diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderIT.java index f082bf419d..7969d25180 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderIT.java @@ -211,8 +211,9 @@ void multipleInlineResourcesEndToEnd() { } @Test - void invalidViewDefinitionReturns4xxError() { - // ViewDefinition missing required 'resource' field. + void invalidViewDefinitionReturns422() { + // A well-formed request carrying a semantically invalid ViewDefinition (missing the required + // 'resource' field) returns 422 Unprocessable Entity, not a generic 4xx (US7, FR-020). final Map invalidView = new HashMap<>(); invalidView.put("resourceType", "ViewDefinition"); invalidView.put("name", "invalid_view"); @@ -238,7 +239,260 @@ void invalidViewDefinitionReturns4xxError() { .bodyValue(gson.toJson(parameters)) .exchange() .expectStatus() - .is4xxClientError(); + .isEqualTo(422); + } + + // ------------------------------------------------------------------------- + // Unsupported request rejection tests (US1) + // ------------------------------------------------------------------------- + + @Test + void unsupportedFormatReturns400NamingTheFormat() { + final String viewJson = createSimplePatientView(); + final Patient patient = new Patient(); + patient.setId("p1"); + patient.addName().setFamily("Smith"); + final String patientJson = fhirContext.newJsonParser().encodeResourceToString(patient); + + final String parametersJson = + createParametersWithInlineResourcesJson(viewJson, "parquet", List.of(patientJson)); + + final byte[] body = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$viewdefinition-run") + .header("Content-Type", "application/fhir+json") + .bodyValue(parametersJson) + .exchange() + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + + assertThat(new String(body, StandardCharsets.UTF_8)).contains("parquet"); + } + + @Test + void sourceParameterReturns400AtSystemLevel() { + sourceRejectedAtUri("http://localhost:" + port + "/fhir/$viewdefinition-run"); + } + + @Test + void sourceParameterReturns400AtTypeLevel() { + sourceRejectedAtUri("http://localhost:" + port + "/fhir/ViewDefinition/$run"); + } + + @Test + void sourceParameterReturns400AtInstanceLevelOverGet() { + // The source rejection happens before the stored ViewDefinition is read, so it applies even + // for an id that would otherwise be a 404. + final byte[] body = + webTestClient + .get() + .uri( + "http://localhost:" + + port + + "/fhir/ViewDefinition/some-view/$run?source=s3://bucket/data") + .header("Accept", "application/x-ndjson") + .exchange() + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + + assertThat(new String(body, StandardCharsets.UTF_8)).contains("source"); + } + + /** Posts a Parameters body carrying a source part to the given URI and asserts a 400. */ + private void sourceRejectedAtUri(@Nonnull final String uri) { + final String viewJson = createSimplePatientView(); + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final List> parameterList = new ArrayList<>(); + + final Map viewParam = new LinkedHashMap<>(); + viewParam.put("name", "viewResource"); + viewParam.put("resource", gson.fromJson(viewJson, Map.class)); + parameterList.add(viewParam); + + final Map sourceParam = new LinkedHashMap<>(); + sourceParam.put("name", "source"); + sourceParam.put("valueString", "s3://bucket/data"); + parameterList.add(sourceParam); + + parameters.put("parameter", parameterList); + + final byte[] body = + webTestClient + .post() + .uri(uri) + .header("Content-Type", "application/fhir+json") + .bodyValue(gson.toJson(parameters)) + .exchange() + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + + assertThat(new String(body, StandardCharsets.UTF_8)).contains("source"); + } + + // ------------------------------------------------------------------------- + // viewReference tests (US4) + // ------------------------------------------------------------------------- + + @Test + void unresolvedViewReferenceReturns404AtSystemLevel() { + unresolvedViewReferenceReturns404("http://localhost:" + port + "/fhir/$viewdefinition-run"); + } + + @Test + void unresolvedViewReferenceReturns404AtTypeLevel() { + unresolvedViewReferenceReturns404("http://localhost:" + port + "/fhir/ViewDefinition/$run"); + } + + /** Posts a viewReference that does not resolve and asserts a 404. */ + private void unresolvedViewReferenceReturns404(@Nonnull final String uri) { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final List> parameterList = new ArrayList<>(); + + final Map referenceValue = new LinkedHashMap<>(); + referenceValue.put("reference", "ViewDefinition/does-not-exist"); + final Map viewReferenceParam = new LinkedHashMap<>(); + viewReferenceParam.put("name", "viewReference"); + viewReferenceParam.put("valueReference", referenceValue); + parameterList.add(viewReferenceParam); + + parameters.put("parameter", parameterList); + + webTestClient + .post() + .uri(uri) + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/x-ndjson") + .bodyValue(gson.toJson(parameters)) + .exchange() + .expectStatus() + .isNotFound(); + } + + // ------------------------------------------------------------------------- + // Reference-typed patient cardinality (US5) + // ------------------------------------------------------------------------- + + @Test + void moreThanOnePatientReturns400() { + final String viewJson = createSimplePatientView(); + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final List> parameterList = new ArrayList<>(); + + final Map viewParam = new LinkedHashMap<>(); + viewParam.put("name", "viewResource"); + viewParam.put("resource", gson.fromJson(viewJson, Map.class)); + parameterList.add(viewParam); + + for (final String id : List.of("Patient/1", "Patient/2")) { + final Map referenceValue = new LinkedHashMap<>(); + referenceValue.put("reference", id); + final Map patientParam = new LinkedHashMap<>(); + patientParam.put("name", "patient"); + patientParam.put("valueReference", referenceValue); + parameterList.add(patientParam); + } + + parameters.put("parameter", parameterList); + + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$viewdefinition-run") + .header("Content-Type", "application/fhir+json") + .bodyValue(gson.toJson(parameters)) + .exchange() + .expectStatus() + .isBadRequest(); + } + + // ------------------------------------------------------------------------- + // Bundle unwrapping in the resource parameter (US6) + // ------------------------------------------------------------------------- + + @Test + void bundleResourceUnwrapsToOneRowPerEntry() { + final String viewJson = createSimplePatientView(); + final String bundleJson = patientBundleJson("b1", "b2", "b3"); + + final String parametersJson = + createParametersWithInlineResourcesJson( + viewJson, "application/x-ndjson", List.of(bundleJson)); + + final EntityExchangeResult result = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$viewdefinition-run") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/x-ndjson") + .bodyValue(parametersJson) + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult(); + + final String responseBody = new String(result.getResponseBodyContent(), StandardCharsets.UTF_8); + final String[] lines = responseBody.trim().split("\n"); + assertThat(lines).hasSize(3); + assertThat(responseBody).contains("b1").contains("b2").contains("b3"); + } + + @Test + void emptyBundleContributesNoRowsAndDoesNotError() { + final String viewJson = createSimplePatientView(); + final String emptyBundle = patientBundleJson(); + final Patient standalone = new Patient(); + standalone.setId("standalone"); + standalone.addName().setFamily("Standalone"); + final String standaloneJson = fhirContext.newJsonParser().encodeResourceToString(standalone); + + final String parametersJson = + createParametersWithInlineResourcesJson( + viewJson, "application/x-ndjson", List.of(emptyBundle, standaloneJson)); + + final EntityExchangeResult result = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$viewdefinition-run") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/x-ndjson") + .bodyValue(parametersJson) + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult(); + + final String responseBody = new String(result.getResponseBodyContent(), StandardCharsets.UTF_8); + final String[] lines = responseBody.trim().split("\n"); + assertThat(lines).hasSize(1); + assertThat(responseBody).contains("standalone"); + } + + /** Builds a Bundle JSON containing one Patient per supplied id. */ + @Nonnull + private String patientBundleJson(@Nonnull final String... patientIds) { + final org.hl7.fhir.r4.model.Bundle bundle = new org.hl7.fhir.r4.model.Bundle(); + bundle.setType(org.hl7.fhir.r4.model.Bundle.BundleType.COLLECTION); + for (final String id : patientIds) { + final Patient patient = new Patient(); + patient.setId(id); + patient.addName().setFamily("Family-" + id); + bundle.addEntry().setResource(patient); + } + return fhirContext.newJsonParser().encodeResourceToString(bundle); } // ------------------------------------------------------------------------- diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderTest.java index 8a196d3bd8..ec92afd7d5 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionRunProviderTest.java @@ -22,6 +22,7 @@ import au.csiro.pathling.operations.compartment.GroupMemberService; import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import au.csiro.pathling.read.ReadExecutor; import au.csiro.pathling.test.SpringBootUnitTest; import au.csiro.pathling.util.FhirServerTestConfiguration; import ca.uhn.fhir.context.FhirContext; @@ -58,7 +59,8 @@ FhirServerTestConfiguration.class, PatientCompartmentService.class, GroupMemberService.class, - ViewExecutionHelper.class + ViewExecutionHelper.class, + ReadExecutor.class }) @SpringBootUnitTest @TestInstance(TestInstance.Lifecycle.PER_METHOD) @@ -104,6 +106,7 @@ void outputFormatReturnsCorrectContentType( provider.run( viewResource, + null, formatParam, null, null, @@ -111,6 +114,7 @@ void outputFormatReturnsCorrectContentType( null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -126,6 +130,7 @@ void csvOutputIncludesHeaderByDefault() throws IOException { provider.run( viewResource, + null, "text/csv", null, null, @@ -133,6 +138,7 @@ void csvOutputIncludesHeaderByDefault() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -152,6 +158,7 @@ void csvOutputExcludesHeaderWhenFalse() throws IOException { provider.run( viewResource, + null, "text/csv", new BooleanType(false), null, @@ -159,6 +166,7 @@ void csvOutputExcludesHeaderWhenFalse() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -182,7 +190,9 @@ void defaultFormatIsNdjson() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -214,6 +224,8 @@ void invalidViewDefinitionThrowsException() { null, null, null, + null, + null, mockRequestDetails(null), response)) .isInstanceOf(Exception.class); @@ -227,6 +239,7 @@ void viewDefinitionWithMultipleColumnsProducesCorrectOutput() throws IOException provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -234,6 +247,7 @@ void viewDefinitionWithMultipleColumnsProducesCorrectOutput() throws IOException null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -260,6 +274,7 @@ void limitRestrictsRowCount() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, new IntegerType(2), @@ -267,6 +282,7 @@ void limitRestrictsRowCount() throws IOException { null, null, patients, + null, mockRequestDetails(null), response); @@ -287,6 +303,7 @@ void noLimitReturnsAllRows() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -294,6 +311,7 @@ void noLimitReturnsAllRows() throws IOException { null, null, patients, + null, mockRequestDetails(null), response); @@ -314,6 +332,7 @@ void inlineResourcesUsedInsteadOfDeltaLake() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -321,6 +340,7 @@ void inlineResourcesUsedInsteadOfDeltaLake() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -338,6 +358,7 @@ void multipleInlineResourcesOfSameType() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -345,6 +366,7 @@ void multipleInlineResourcesOfSameType() throws IOException { null, null, patients, + null, mockRequestDetails(null), response); @@ -366,6 +388,7 @@ void inlineResourcesFilteredByViewResourceType() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -373,6 +396,7 @@ void inlineResourcesFilteredByViewResourceType() throws IOException { null, null, resources, + null, mockRequestDetails(null), response); @@ -399,7 +423,9 @@ void invalidInlineResourceThrowsException() { null, null, null, + null, invalidResources, + null, requestDetails, response)) .isInstanceOf(InvalidRequestException.class) @@ -414,6 +440,141 @@ void invalidInlineResourceThrowsException() { // ObjectDataSource does not implement QueryableDataSource. These tests would require // integration tests with a full Delta Lake setup. + // ------------------------------------------------------------------------- + // Unsupported request rejection tests (US1) + // ------------------------------------------------------------------------- + + @Test + void rejectsUnsupportedFormatNamingValue() { + final MockHttpServletResponse response = new MockHttpServletResponse(); + final IBaseResource viewResource = parseViewResource(createSimplePatientView()); + final String inlinePatient = createPatientJson("test-1", "Smith"); + + assertThatThrownBy( + () -> + provider.run( + viewResource, + null, + "parquet", + null, + null, + null, + null, + null, + List.of(inlinePatient), + null, + mockRequestDetails(null), + response)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("parquet"); + } + + @Test + void rejectsSourceParameterWhenSupplied() { + final MockHttpServletResponse response = new MockHttpServletResponse(); + final IBaseResource viewResource = parseViewResource(createSimplePatientView()); + final String inlinePatient = createPatientJson("test-1", "Smith"); + + assertThatThrownBy( + () -> + provider.run( + viewResource, + null, + null, + null, + null, + null, + null, + null, + List.of(inlinePatient), + "s3://bucket/data", + mockRequestDetails(null), + response)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("source"); + } + + // ------------------------------------------------------------------------- + // View selection tests (US4): viewResource / viewReference exclusivity and presence + // ------------------------------------------------------------------------- + + @Test + void rejectsBothViewResourceAndViewReference() { + final MockHttpServletResponse response = new MockHttpServletResponse(); + final IBaseResource viewResource = parseViewResource(createSimplePatientView()); + + assertThatThrownBy( + () -> + provider.run( + viewResource, + new Reference("ViewDefinition/some-view"), + null, + null, + null, + null, + null, + null, + null, + null, + mockRequestDetails(null), + response)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("viewResource") + .hasMessageContaining("viewReference"); + } + + @Test + void rejectsNeitherViewResourceNorViewReference() { + final MockHttpServletResponse response = new MockHttpServletResponse(); + + assertThatThrownBy( + () -> + provider.run( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + mockRequestDetails(null), + response)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("viewResource") + .hasMessageContaining("viewReference"); + } + + // ------------------------------------------------------------------------- + // Patient cardinality tests (US5) + // ------------------------------------------------------------------------- + + @Test + void rejectsMoreThanOnePatient() { + final MockHttpServletResponse response = new MockHttpServletResponse(); + final IBaseResource viewResource = parseViewResource(createSimplePatientView()); + + assertThatThrownBy( + () -> + provider.run( + viewResource, + null, + null, + null, + null, + List.of(new Reference("Patient/1"), new Reference("Patient/2")), + null, + null, + null, + null, + mockRequestDetails(null), + response)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("patient"); + } + // ------------------------------------------------------------------------- // Error handling tests // ------------------------------------------------------------------------- @@ -427,7 +588,18 @@ void nullResponseThrowsInvalidRequestException() { assertThatThrownBy( () -> provider.run( - viewResource, null, null, null, null, null, null, null, requestDetails, null)) + viewResource, + null, + null, + null, + null, + null, + null, + null, + null, + null, + requestDetails, + null)) .isInstanceOf(InvalidRequestException.class) .hasMessageContaining("HTTP response is required"); } @@ -450,7 +622,9 @@ void choiceElementWithoutTypeThrowsDescriptiveError() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(null), response)) .isInstanceOf(InvalidRequestException.class) @@ -470,6 +644,7 @@ void ndjsonRowContainsExpectedFields() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -477,6 +652,7 @@ void ndjsonRowContainsExpectedFields() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -503,6 +679,7 @@ void ndjsonHandlesNullValues() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -510,6 +687,7 @@ void ndjsonHandlesNullValues() throws IOException { null, null, List.of(patientJson), + null, mockRequestDetails(null), response); @@ -530,6 +708,7 @@ void csvOutputProducesValidFormat() throws IOException { provider.run( viewResource, + null, "text/csv", null, null, @@ -537,6 +716,7 @@ void csvOutputProducesValidFormat() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -588,6 +768,7 @@ void jsonOutputReturnsArrayWithSingleRow() throws IOException { provider.run( viewResource, + null, "json", null, null, @@ -595,6 +776,7 @@ void jsonOutputReturnsArrayWithSingleRow() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), response); @@ -619,6 +801,7 @@ void jsonOutputReturnsArrayWithMultipleRows() throws IOException { provider.run( viewResource, + null, "json", null, null, @@ -626,6 +809,7 @@ void jsonOutputReturnsArrayWithMultipleRows() throws IOException { null, null, patients, + null, mockRequestDetails(null), response); @@ -645,6 +829,7 @@ void jsonRowFormatMatchesNdjsonRowFormat() throws IOException { provider.run( viewResource, + null, "ndjson", null, null, @@ -652,6 +837,7 @@ void jsonRowFormatMatchesNdjsonRowFormat() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), ndjsonResponse); @@ -663,6 +849,7 @@ void jsonRowFormatMatchesNdjsonRowFormat() throws IOException { final MockHttpServletResponse jsonResponse = new MockHttpServletResponse(); provider.run( viewResource, + null, "json", null, null, @@ -670,6 +857,7 @@ void jsonRowFormatMatchesNdjsonRowFormat() throws IOException { null, null, List.of(inlinePatient), + null, mockRequestDetails(null), jsonResponse); @@ -695,6 +883,7 @@ void jsonOutputRespectsLimit() throws IOException { provider.run( viewResource, + null, "json", null, new IntegerType(2), @@ -702,6 +891,7 @@ void jsonOutputRespectsLimit() throws IOException { null, null, patients, + null, mockRequestDetails(null), response); @@ -724,6 +914,7 @@ void acceptHeaderDeterminesOutputFormat() { provider.run( viewResource, + null, null, // No _format parameter. null, null, @@ -731,6 +922,7 @@ void acceptHeaderDeterminesOutputFormat() { null, null, List.of(inlinePatient), + null, mockRequestDetails("text/csv"), response); @@ -746,6 +938,7 @@ void formatParamOverridesAcceptHeader() { provider.run( viewResource, + null, "ndjson", // Explicit _format parameter. null, null, @@ -753,6 +946,7 @@ void formatParamOverridesAcceptHeader() { null, null, List.of(inlinePatient), + null, mockRequestDetails("text/csv"), // Different Accept header. response); @@ -775,7 +969,9 @@ void wildcardAcceptHeaderDefaultsToNdjson() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails("*/*"), response); @@ -796,6 +992,7 @@ void constantsWithValueCodeAreParsed() throws IOException { provider.run( viewResource, + null, "application/x-ndjson", null, null, @@ -803,6 +1000,7 @@ void constantsWithValueCodeAreParsed() throws IOException { null, null, List.of(matchingPatient, nonMatchingPatient), + null, mockRequestDetails(null), response); diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperResolutionTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperResolutionTest.java new file mode 100644 index 0000000000..bfff71aaf4 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperResolutionTest.java @@ -0,0 +1,159 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.view; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; +import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; +import au.csiro.pathling.library.PathlingContext; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.compartment.GroupMemberService; +import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.test.SpringBootUnitTest; +import au.csiro.pathling.util.CustomObjectDataSource; +import au.csiro.pathling.util.FhirServerTestConfiguration; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import jakarta.annotation.Nonnull; +import java.util.List; +import org.apache.spark.sql.SparkSession; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; + +/** + * Tests for {@link ViewExecutionHelper#resolveViewInput} and {@link + * ViewExecutionHelper#readStoredViewDefinition}, exercising {@code viewReference} resolution + * against a stored ViewDefinition (US4, FR-009..FR-013). Uses a {@link CustomObjectDataSource} + * backing the read path, because newly created ViewDefinitions are not visible to the + * integration-test server's cached Delta Lake source. + * + * @author John Grimes + */ +@Import({ + FhirServerTestConfiguration.class, + PatientCompartmentService.class, + GroupMemberService.class +}) +@SpringBootUnitTest +class ViewExecutionHelperResolutionTest { + + @Autowired private SparkSession sparkSession; + @Autowired private PathlingContext pathlingContext; + @Autowired private FhirEncoders fhirEncoders; + @Autowired private FhirContext fhirContext; + @Autowired private QueryableDataSource deltaLake; + @Autowired private PatientCompartmentService patientCompartmentService; + @Autowired private GroupMemberService groupMemberService; + @Autowired private ServerConfiguration serverConfiguration; + + private ViewExecutionHelper helper; + + @BeforeEach + void setUp() { + // Back the read path with a CustomObjectDataSource that holds a stored ViewDefinition. + final List stored = + List.of(createSimpleViewDefinition("stored-view", "stored_view", "Patient")); + final CustomObjectDataSource dataSource = + new CustomObjectDataSource(sparkSession, pathlingContext, fhirEncoders, stored); + final ReadExecutor readExecutor = new ReadExecutor(dataSource, fhirEncoders); + + helper = + new ViewExecutionHelper( + sparkSession, + deltaLake, + fhirContext, + fhirEncoders, + patientCompartmentService, + groupMemberService, + serverConfiguration, + readExecutor); + } + + @Test + void resolvesViewReferenceToStoredViewDefinition() { + // A viewReference of the form ViewDefinition/{id} resolves to the stored ViewDefinition. + final IBaseResource resolved = + helper.resolveViewInput(null, new Reference("ViewDefinition/stored-view")); + + assertThat(resolved).isInstanceOf(ViewDefinitionResource.class); + assertThat(((ViewDefinitionResource) resolved).getIdElement().getIdPart()) + .isEqualTo("stored-view"); + } + + @Test + void returnsInlineViewResourceWhenSupplied() { + final ViewDefinitionResource inline = + createSimpleViewDefinition("inline", "inline_view", "Patient"); + assertThat(helper.resolveViewInput(inline, null)).isSameAs(inline); + } + + @Test + void rejectsUnresolvedViewReferenceWithNotFound() { + assertThatThrownBy( + () -> helper.resolveViewInput(null, new Reference("ViewDefinition/does-not-exist"))) + .isInstanceOf(ResourceNotFoundException.class); + } + + @Test + void rejectsBothViewResourceAndViewReference() { + final ViewDefinitionResource inline = + createSimpleViewDefinition("inline", "inline_view", "Patient"); + assertThatThrownBy( + () -> helper.resolveViewInput(inline, new Reference("ViewDefinition/stored-view"))) + .isInstanceOf(InvalidRequestException.class); + } + + @Test + void rejectsNeitherViewResourceNorViewReference() { + assertThatThrownBy(() -> helper.resolveViewInput(null, null)) + .isInstanceOf(InvalidRequestException.class); + } + + /** Builds a minimal stored ViewDefinition over the given resource type. */ + @Nonnull + private ViewDefinitionResource createSimpleViewDefinition( + @Nonnull final String id, @Nonnull final String name, @Nonnull final String resource) { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(id); + view.setName(new StringType(name)); + view.setResource(new CodeType(resource)); + view.setStatus(new CodeType("active")); + + final SelectComponent select = new SelectComponent(); + final ColumnComponent column = new ColumnComponent(); + column.setName(new StringType("id")); + column.setPath(new StringType("id")); + select.getColumn().add(column); + view.getSelect().add(select); + + return view; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java index cd270a07d4..e41cb221a0 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java @@ -18,7 +18,9 @@ package au.csiro.pathling.operations.view; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import org.junit.jupiter.api.Test; /** @@ -43,12 +45,6 @@ void parsesNdjsonContentType() { .isEqualTo(ViewExportFormat.NDJSON); } - @Test - void parsesNdjsonFhirContentType() { - assertThat(ViewExportFormat.fromString("application/fhir+ndjson")) - .isEqualTo(ViewExportFormat.NDJSON); - } - @Test void parsesCsvCode() { assertThat(ViewExportFormat.fromString("csv")).isEqualTo(ViewExportFormat.CSV); @@ -71,8 +67,19 @@ void parsesParquetContentType() { } @Test - void defaultsToNdjsonForUnknown() { - assertThat(ViewExportFormat.fromString("unknown")).isEqualTo(ViewExportFormat.NDJSON); + void rejectsUnknownNamingValue() { + // An explicit unsupported format is rejected with the unsupported value named. + assertThatThrownBy(() -> ViewExportFormat.fromString("unknown")) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("unknown"); + } + + @Test + void rejectsJsonAsUnsupported() { + // json is only RECOMMENDED by the spec and is not supported by the export operation. + assertThatThrownBy(() -> ViewExportFormat.fromString("json")) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("json"); } @Test diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java index 0afb80539b..b295a1ee43 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java @@ -18,7 +18,9 @@ package au.csiro.pathling.operations.view; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import org.junit.jupiter.api.Test; /** @@ -63,11 +65,6 @@ void fromStringParsesJsonContentType() { assertThat(ViewOutputFormat.fromString("application/json")).isEqualTo(ViewOutputFormat.JSON); } - @Test - void fromStringDefaultsToNdjsonForUnknown() { - assertThat(ViewOutputFormat.fromString("unknown")).isEqualTo(ViewOutputFormat.NDJSON); - } - @Test void fromStringDefaultsToNdjsonForEmptyString() { assertThat(ViewOutputFormat.fromString("")).isEqualTo(ViewOutputFormat.NDJSON); @@ -78,6 +75,44 @@ void fromStringDefaultsToNdjsonForNull() { assertThat(ViewOutputFormat.fromString(null)).isEqualTo(ViewOutputFormat.NDJSON); } + // ------------------------------------------------------------------------- + // fromStringStrict parsing tests (used for the explicit _format parameter) + // ------------------------------------------------------------------------- + + @Test + void fromStringStrictParsesSupportedCodes() { + assertThat(ViewOutputFormat.fromStringStrict("ndjson")).isEqualTo(ViewOutputFormat.NDJSON); + assertThat(ViewOutputFormat.fromStringStrict("csv")).isEqualTo(ViewOutputFormat.CSV); + assertThat(ViewOutputFormat.fromStringStrict("json")).isEqualTo(ViewOutputFormat.JSON); + } + + @Test + void fromStringStrictParsesSupportedContentTypes() { + assertThat(ViewOutputFormat.fromStringStrict("application/x-ndjson")) + .isEqualTo(ViewOutputFormat.NDJSON); + assertThat(ViewOutputFormat.fromStringStrict("text/csv")).isEqualTo(ViewOutputFormat.CSV); + assertThat(ViewOutputFormat.fromStringStrict("application/json")) + .isEqualTo(ViewOutputFormat.JSON); + } + + @Test + void fromStringStrictDefaultsToNdjsonForNull() { + assertThat(ViewOutputFormat.fromStringStrict(null)).isEqualTo(ViewOutputFormat.NDJSON); + } + + @Test + void fromStringStrictDefaultsToNdjsonForBlank() { + assertThat(ViewOutputFormat.fromStringStrict(" ")).isEqualTo(ViewOutputFormat.NDJSON); + } + + @Test + void fromStringStrictRejectsUnknownNamingValue() { + // An explicit unsupported format is rejected with the unsupported value named. + assertThatThrownBy(() -> ViewOutputFormat.fromStringStrict("parquet")) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("parquet"); + } + // ------------------------------------------------------------------------- // fromAcceptHeader parsing tests // ------------------------------------------------------------------------- diff --git a/server/src/test/java/au/csiro/pathling/security/SecurityEnabledViewOperationTest.java b/server/src/test/java/au/csiro/pathling/security/SecurityEnabledViewOperationTest.java index 0321fe68e8..339e307050 100644 --- a/server/src/test/java/au/csiro/pathling/security/SecurityEnabledViewOperationTest.java +++ b/server/src/test/java/au/csiro/pathling/security/SecurityEnabledViewOperationTest.java @@ -28,6 +28,7 @@ import au.csiro.pathling.operations.compartment.PatientCompartmentService; import au.csiro.pathling.operations.view.ViewDefinitionRunProvider; import au.csiro.pathling.operations.view.ViewExecutionHelper; +import au.csiro.pathling.read.ReadExecutor; import au.csiro.pathling.util.FhirServerTestConfiguration; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; @@ -69,7 +70,8 @@ FhirServerTestConfiguration.class, PatientCompartmentService.class, GroupMemberService.class, - ViewExecutionHelper.class + ViewExecutionHelper.class, + ReadExecutor.class }) class SecurityEnabledViewOperationTest extends SecurityTest { @@ -109,7 +111,9 @@ void viewRunDeniedWithoutReadAuthority() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(), response)) .isExactlyInstanceOf(AccessDeniedError.class) @@ -140,7 +144,9 @@ void viewRunSucceedsWithSpecificReadAuthority() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(), response)); @@ -170,7 +176,9 @@ void viewRunSucceedsWithWildcardReadAuthority() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(), response)); @@ -200,7 +208,9 @@ void viewRunDeniedWithWrongResourceTypeAuthority() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(), response)) .isExactlyInstanceOf(AccessDeniedError.class) @@ -230,7 +240,9 @@ void viewRunDeniedWithoutOperationAuthority() { null, null, null, + null, List.of(inlinePatient), + null, mockRequestDetails(), response)) .isExactlyInstanceOf(AccessDeniedError.class) @@ -260,7 +272,9 @@ void viewRunSucceedsWithMatchingResourceType() { null, null, null, + null, List.of(inlineObservation), + null, mockRequestDetails(), response)); From 5c1434efe227f339ea2bca5b11b3a6f463f6e4f7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:13:02 +1000 Subject: [PATCH 002/162] feat: Align SQL on FHIR export operation to the spec Support view.viewReference resolution to a stored ViewDefinition with per-part exclusivity and presence checks. Rewrite the completion manifest to the SQL on FHIR shape (exportId, status, echoed clientTrackingId and _format, export timing fields, and one output per view with name and location parts), dropping the Bulk Data-style fields. Return a Parameters kick-off acknowledgement for SQL on FHIR async operations while leaving the Bulk Data kick-off body unchanged. --- .../au/csiro/pathling/async/AsyncAspect.java | 38 ++- .../view/ViewDefinitionExportProvider.java | 36 ++- .../view/ViewDefinitionExportResponse.java | 94 ++++-- .../view/ViewDefinitionExportProviderIT.java | 294 ++++++++++++++++++ .../ViewDefinitionExportProviderTest.java | 6 +- .../ViewDefinitionExportResponseTest.java | 247 +++++---------- 6 files changed, 508 insertions(+), 207 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java b/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java index d583e0411c..3a22c5b256 100644 --- a/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java +++ b/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java @@ -24,6 +24,7 @@ import au.csiro.pathling.errors.DiagnosticContext; import au.csiro.pathling.errors.ErrorHandlingInterceptor; import au.csiro.pathling.errors.ErrorReportingInterceptor; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; @@ -41,10 +42,13 @@ import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.OperationOutcome; import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; import org.hl7.fhir.r4.model.OperationOutcome.IssueType; import org.hl7.fhir.r4.model.OperationOutcome.OperationOutcomeIssueComponent; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.StringType; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -134,14 +138,28 @@ protected IBaseResource maybeExecuteAsynchronously( // PreAsyncValidation. Set some values to prevent NPEs. result = new PreAsyncValidationResult<>(new Object(), List.of()); } - processRequestAsynchronously(joinPoint, requestDetails, result, spark, asyncSupported); + final Job job = + processRequestAsynchronously(joinPoint, requestDetails, result, spark, asyncSupported); + + if (asyncSupported.redirectOnComplete()) { + // For SQL on FHIR async operations, the kick-off body is a Parameters acknowledgement + // (status=accepted, exportId) returned with a 202 status, rather than an OperationOutcome. + // The Content-Location header is already set by processRequestAsynchronously. + final HttpServletResponse response = requestDetails.getServletResponse(); + if (response != null) { + response.setStatus(Constants.STATUS_HTTP_202_ACCEPTED); + } + return buildAcceptedAcknowledgement(job.getId()); + } + + // FHIR Bulk Data operations keep the existing OperationOutcome kick-off body. throw new ProcessingNotCompletedException("Accepted", buildOperationOutcome(result)); } else { return (IBaseResource) joinPoint.proceed(); } } - private void processRequestAsynchronously( + private Job processRequestAsynchronously( @Nonnull final ProceedingJoinPoint joinPoint, @Nonnull final ServletRequestDetails requestDetails, @Nonnull final PreAsyncValidationResult preAsyncValidationResult, @@ -229,6 +247,22 @@ private void processRequestAsynchronously( final String asyncEtag = "W/\"~" + serverInstanceId.getId() + "." + hashJobId(job.getId()) + "\""; response.setHeader("ETag", asyncEtag); + return job; + } + + /** + * Builds the SQL on FHIR kick-off acknowledgement: a {@code Parameters} resource carrying {@code + * status=accepted} and the {@code exportId}. + * + * @param jobId the server-assigned job identifier + * @return the acknowledgement Parameters resource + */ + @Nonnull + private static Parameters buildAcceptedAcknowledgement(@Nonnull final String jobId) { + final Parameters parameters = new Parameters(); + parameters.addParameter().setName("status").setValue(new CodeType("accepted")); + parameters.addParameter().setName("exportId").setValue(new StringType(jobId)); + return parameters; } /** diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java index 92d025255c..99cd5dc528 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java @@ -47,6 +47,7 @@ import com.google.gson.JsonSyntaxException; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -64,6 +65,7 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.InstantType; import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; @@ -98,6 +100,8 @@ public class ViewDefinitionExportProvider @Nonnull private final GroupMemberService groupMemberService; + @Nonnull private final ViewExecutionHelper viewExecutionHelper; + @Nonnull private final Gson gson; /** @@ -111,7 +115,10 @@ public class ViewDefinitionExportProvider * @param fhirContext the FHIR context * @param deltaLake the queryable data source * @param groupMemberService the group member service + * @param viewExecutionHelper the helper used to resolve a view.viewReference to a stored + * ViewDefinition via the shared read path */ + @SuppressWarnings("java:S107") @Autowired public ViewDefinitionExportProvider( @Nonnull final ViewDefinitionExportExecutor executor, @@ -121,7 +128,8 @@ public ViewDefinitionExportProvider( @Nonnull final ServerConfiguration serverConfiguration, @Nonnull final FhirContext fhirContext, @Nonnull final QueryableDataSource deltaLake, - @Nonnull final GroupMemberService groupMemberService) { + @Nonnull final GroupMemberService groupMemberService, + @Nonnull final ViewExecutionHelper viewExecutionHelper) { this.executor = executor; this.jobRegistry = jobRegistry; this.requestTagFactory = requestTagFactory; @@ -130,6 +138,7 @@ public ViewDefinitionExportProvider( this.fhirContext = fhirContext; this.deltaLake = deltaLake; this.groupMemberService = groupMemberService; + this.viewExecutionHelper = viewExecutionHelper; this.gson = ViewDefinitionGson.create(); } @@ -234,10 +243,13 @@ public Parameters export( final ViewDefinitionExportResponse response = new ViewDefinitionExportResponse( - exportRequest.originalRequest(), exportRequest.serverBaseUrl(), outputs, - serverConfiguration.getAuth().isEnabled()); + ownJob.getId(), + exportRequest.clientTrackingId(), + exportRequest.format().getCode(), + ownJob.getStartTime(), + Instant.now()); return response.toOutput(); } @@ -366,21 +378,27 @@ private List extractViewInputsFromRequest( if ("view".equals(param.getName())) { String viewName = null; IBaseResource viewResource = null; + Reference viewReference = null; - // Extract name and viewResource from the nested parts. + // Extract name, viewResource, and viewReference from the nested parts. for (final Parameters.ParametersParameterComponent part : param.getPart()) { if ("name".equals(part.getName()) && part.getValue() != null) { viewName = part.getValue().primitiveValue(); } else if ("viewResource".equals(part.getName()) && part.getResource() != null) { viewResource = part.getResource(); + } else if ("viewReference".equals(part.getName()) + && part.getValue() instanceof final Reference reference) { + viewReference = reference; } } - if (viewResource != null) { - final FhirView fhirView = parseViewDefinition(viewResource, viewIndex); - views.add(new ViewInput(viewName, fhirView)); - viewIndex++; - } + // Each view part must supply exactly one of viewResource or viewReference; resolveViewInput + // enforces the exclusivity (400), presence (400), and reference resolution (404). + final IBaseResource resolved = + viewExecutionHelper.resolveViewInput(viewResource, viewReference); + final FhirView fhirView = parseViewDefinition(resolved, viewIndex); + views.add(new ViewInput(viewName, fhirView)); + viewIndex++; } } diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java index 299090ed38..afbb20f25e 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java @@ -20,51 +20,80 @@ import au.csiro.pathling.OperationResponse; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.net.URISyntaxException; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; import java.util.List; import lombok.Getter; import org.apache.http.client.utils.URIBuilder; -import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.IntegerType; import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; import org.hl7.fhir.r4.model.StringType; import org.hl7.fhir.r4.model.UriType; /** - * Represents the response from a ViewDefinition export operation, containing the export manifest. + * Represents the completion manifest returned at the result URL of a {@code $viewdefinition-export} + * operation. The manifest follows the SQL on FHIR shape: a required {@code exportId} and {@code + * status}, the echoed {@code clientTrackingId} and {@code _format}, the export timing fields, and + * one {@code output} per exported view with a {@code name} and one or more {@code location} + * download URLs. * * @author John Grimes */ @Getter public class ViewDefinitionExportResponse implements OperationResponse { - @Nonnull private final String kickOffRequestUrl; - @Nonnull private final String serverBaseUrl; @Nonnull private final List outputs; - /** Whether an access token is required to retrieve results. */ - private final boolean requiresAccessToken; + /** The server-assigned export job identifier. */ + @Nonnull private final String exportId; + + /** The client-supplied tracking identifier, echoed only when present. */ + @Nullable private final String clientTrackingId; + + /** The effective output format code (e.g. {@code ndjson}). */ + @Nonnull private final String format; + + /** The time the export job was created (kick-off time). */ + @Nonnull private final Instant exportStartTime; + + /** The time the manifest was built on completion. */ + @Nonnull private final Instant exportEndTime; /** * Creates a new ViewDefinitionExportResponse. * - * @param kickOffRequestUrl the original export request URL (used in the manifest) * @param serverBaseUrl the FHIR server base URL (used for constructing result URLs) * @param outputs the list of view export outputs - * @param requiresAccessToken whether access token is required to retrieve results + * @param exportId the server-assigned export job identifier + * @param clientTrackingId the client-supplied tracking identifier, or null if none was supplied + * @param format the effective output format code + * @param exportStartTime the export job creation (kick-off) time + * @param exportEndTime the time the manifest is built on completion */ + @SuppressWarnings("java:S107") public ViewDefinitionExportResponse( - @Nonnull final String kickOffRequestUrl, @Nonnull final String serverBaseUrl, @Nonnull final List outputs, - final boolean requiresAccessToken) { - this.kickOffRequestUrl = kickOffRequestUrl; + @Nonnull final String exportId, + @Nullable final String clientTrackingId, + @Nonnull final String format, + @Nonnull final Instant exportStartTime, + @Nonnull final Instant exportEndTime) { this.serverBaseUrl = serverBaseUrl; this.outputs = outputs; - this.requiresAccessToken = requiresAccessToken; + this.exportId = exportId; + this.clientTrackingId = clientTrackingId; + this.format = format; + this.exportStartTime = exportStartTime; + this.exportEndTime = exportEndTime; } @Nonnull @@ -76,27 +105,44 @@ public Parameters toOutput() { final String normalizedBaseUrl = serverBaseUrl.endsWith("/") ? serverBaseUrl : serverBaseUrl + "/"; - // Add transactionTime parameter. - parameters.addParameter().setName("transactionTime").setValue(InstantType.now()); + parameters.addParameter().setName("exportId").setValue(new StringType(exportId)); + parameters.addParameter().setName("status").setValue(new CodeType("completed")); + + // Echo the client tracking id only when one was supplied at kick-off. + if (clientTrackingId != null && !clientTrackingId.isBlank()) { + parameters + .addParameter() + .setName("clientTrackingId") + .setValue(new StringType(clientTrackingId)); + } + + parameters.addParameter().setName("_format").setValue(new CodeType(format)); - // Add request parameter. - parameters.addParameter().setName("request").setValue(new UriType(kickOffRequestUrl)); + parameters + .addParameter() + .setName("exportStartTime") + .setValue(new InstantType(Date.from(exportStartTime))); + parameters + .addParameter() + .setName("exportEndTime") + .setValue(new InstantType(Date.from(exportEndTime))); - // Add requiresAccessToken parameter. + // Whole seconds between start and end, never negative. + final long durationSeconds = + Math.max(0L, Duration.between(exportStartTime, exportEndTime).toSeconds()); parameters .addParameter() - .setName("requiresAccessToken") - .setValue(new BooleanType(requiresAccessToken)); + .setName("exportDuration") + .setValue(new IntegerType((int) durationSeconds)); - // Add output parameters. + // One output per view, with a name and one or more location parts. for (final ViewExportOutput output : outputs) { + final ParametersParameterComponent outputParam = parameters.addParameter().setName("output"); + outputParam.addPart().setName("name").setValue(new StringType(output.name())); for (final String fileUrl : output.fileUrls()) { - final ParametersParameterComponent outputParam = - parameters.addParameter().setName("output"); - outputParam.addPart().setName("name").setValue(new StringType(output.name())); outputParam .addPart() - .setName("url") + .setName("location") .setValue(new UriType(buildResultUrl(normalizedBaseUrl, fileUrl))); } } diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java index 4655dd6f7a..b6a79db993 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java @@ -126,10 +126,304 @@ void kickOffWithMultipleNamedViews() { .exists("Content-Location"); } + // ------------------------------------------------------------------------- + // Kick-off acknowledgement body (US10) + // ------------------------------------------------------------------------- + + @Test + void kickOffBodyIsParametersAcknowledgement() { + final byte[] body = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$viewdefinition-export") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/fhir+json") + .header("Prefer", "respond-async") + .bodyValue(createExportParametersWithNestedView()) + .exchange() + .expectStatus() + .isAccepted() + .expectHeader() + .exists("Content-Location") + .expectBody() + .returnResult() + .getResponseBodyContent(); + + final Map manifest = parseParameters(body); + assertThat(manifest.get("resourceType")).isEqualTo("Parameters"); + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("accepted"); + assertThat(findParamValue(manifest, "exportId", "valueString")).isNotNull(); + } + + // ------------------------------------------------------------------------- + // Synchronous kick-off rejections (US1, US8) + // ------------------------------------------------------------------------- + + @Test + void unsupportedFormatRejectedAtKickOff() { + final Map parameters = parametersWithNestedView(); + addSimpleParameter(parameters, "_format", "valueString", "json"); + + final byte[] body = + kickOff(gson.toJson(parameters)) + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + assertThat(new String(body, java.nio.charset.StandardCharsets.UTF_8)).contains("json"); + } + + @Test + void sourceRejectedAtKickOff() { + final Map parameters = parametersWithNestedView(); + addSimpleParameter(parameters, "source", "valueString", "s3://bucket/data"); + + final byte[] body = + kickOff(gson.toJson(parameters)) + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + assertThat(new String(body, java.nio.charset.StandardCharsets.UTF_8)).contains("source"); + } + + @Test + void viewPartWithBothResourceAndReferenceRejectedAtKickOff() { + final Map viewPart = viewPartWithResource(); + @SuppressWarnings("unchecked") + final List> parts = (List>) viewPart.get("part"); + parts.add(referencePart("viewReference", "ViewDefinition/some-view")); + + final Map parameters = parametersWith(viewPart); + kickOff(gson.toJson(parameters)).expectStatus().isBadRequest(); + } + + @Test + void viewPartWithNeitherResourceNorReferenceRejectedAtKickOff() { + final Map viewPart = new LinkedHashMap<>(); + viewPart.put("name", "view"); + viewPart.put("part", new ArrayList<>(List.of(simplePart("name", "valueString", "empty")))); + + final Map parameters = parametersWith(viewPart); + kickOff(gson.toJson(parameters)).expectStatus().isBadRequest(); + } + + @Test + void unresolvedViewReferenceRejectedAtKickOffWith404() { + final Map viewPart = new LinkedHashMap<>(); + viewPart.put("name", "view"); + viewPart.put( + "part", + new ArrayList<>(List.of(referencePart("viewReference", "ViewDefinition/does-not-exist")))); + + final Map parameters = parametersWith(viewPart); + kickOff(gson.toJson(parameters)).expectStatus().isNotFound(); + } + + // ------------------------------------------------------------------------- + // Completion manifest shape (US9) + // ------------------------------------------------------------------------- + + @Test + void completionManifestHasSpecShape() throws InterruptedException { + final Map parameters = parametersWithNestedView(); + addSimpleParameter(parameters, "clientTrackingId", "valueString", "tracking-xyz"); + + final Map manifest = exportToCompletion(gson.toJson(parameters)); + + assertThat(findParamValue(manifest, "exportId", "valueString")).isNotNull(); + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("completed"); + assertThat(findParamValue(manifest, "clientTrackingId", "valueString")) + .isEqualTo("tracking-xyz"); + assertThat(findParamValue(manifest, "_format", "valueCode")).isEqualTo("ndjson"); + assertThat(findParamValue(manifest, "exportStartTime", "valueInstant")).isNotNull(); + assertThat(findParamValue(manifest, "exportEndTime", "valueInstant")).isNotNull(); + assertThat(findParam(manifest, "exportDuration")).isNotNull(); + + // Each output has a name and one or more location parts; no url part and no Bulk Data fields. + final Map output = findParam(manifest, "output"); + assertThat(output).isNotNull(); + assertThat(findPartValue(output, "name", "valueString")).isNotNull(); + assertThat(findPartValue(output, "location", "valueUri")).contains("$result"); + assertThat(findPart(output, "url")).isNull(); + + assertThat(findParam(manifest, "transactionTime")).isNull(); + assertThat(findParam(manifest, "request")).isNull(); + assertThat(findParam(manifest, "requiresAccessToken")).isNull(); + } + + @Test + void completionManifestOmitsClientTrackingIdWhenAbsent() throws InterruptedException { + final Map manifest = exportToCompletion(createExportParametersWithNestedView()); + assertThat(findParam(manifest, "clientTrackingId")).isNull(); + } + // ------------------------------------------------------------------------- // Helper methods // ------------------------------------------------------------------------- + /** Posts a kick-off request with the async preference and returns the response spec. */ + @Nonnull + private WebTestClient.ResponseSpec kickOff(@Nonnull final String parametersJson) { + return webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$viewdefinition-export") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/fhir+json") + .header("Prefer", "respond-async") + .bodyValue(parametersJson) + .exchange(); + } + + /** Runs an export to completion and returns the parsed result manifest. */ + @Nonnull + private Map exportToCompletion(@Nonnull final String parametersJson) + throws InterruptedException { + final String contentLocation = + kickOff(parametersJson) + .expectStatus() + .isAccepted() + .returnResult(String.class) + .getResponseHeaders() + .getFirst("Content-Location"); + assertThat(contentLocation).isNotNull(); + + String resultLocation = null; + for (int attempt = 0; attempt < 60 && resultLocation == null; attempt++) { + final var poll = + webTestClient + .get() + .uri(contentLocation) + .header("Accept", "application/fhir+json") + .exchange() + .returnResult(String.class); + final HttpStatus status = (HttpStatus) poll.getStatus(); + if (status == HttpStatus.SEE_OTHER) { + resultLocation = poll.getResponseHeaders().getFirst("Location"); + } else if (status == HttpStatus.ACCEPTED) { + Thread.sleep(500); + } else { + throw new AssertionError("Unexpected poll status: " + status); + } + } + assertThat(resultLocation).as("Expected a 303 result location within the timeout").isNotNull(); + + final byte[] body = + webTestClient + .get() + .uri(resultLocation) + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult() + .getResponseBodyContent(); + return parseParameters(body); + } + + @Nonnull + private Map parametersWithNestedView() { + return parametersWith(viewPartWithResource()); + } + + @Nonnull + private Map viewPartWithResource() { + final Map viewParam = new LinkedHashMap<>(); + viewParam.put("name", "view"); + final List> parts = new ArrayList<>(); + final Map viewResourcePart = new LinkedHashMap<>(); + viewResourcePart.put("name", "viewResource"); + viewResourcePart.put("resource", createSimplePatientViewDefinition()); + parts.add(viewResourcePart); + viewParam.put("part", parts); + return viewParam; + } + + @Nonnull + private Map parametersWith(@Nonnull final Map viewParam) { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + parameters.put("parameter", new ArrayList<>(List.of(viewParam))); + return parameters; + } + + @SuppressWarnings("unchecked") + private void addSimpleParameter( + @Nonnull final Map parameters, + @Nonnull final String name, + @Nonnull final String valueKey, + @Nonnull final String value) { + ((List>) parameters.get("parameter")) + .add(simplePart(name, valueKey, value)); + } + + @Nonnull + private Map simplePart( + @Nonnull final String name, @Nonnull final String valueKey, @Nonnull final String value) { + final Map part = new LinkedHashMap<>(); + part.put("name", name); + part.put(valueKey, value); + return part; + } + + @Nonnull + private Map referencePart( + @Nonnull final String name, @Nonnull final String reference) { + final Map part = new LinkedHashMap<>(); + part.put("name", name); + part.put("valueReference", Map.of("reference", reference)); + return part; + } + + @SuppressWarnings("unchecked") + @Nonnull + private Map parseParameters(@Nonnull final byte[] body) { + return gson.fromJson(new String(body, java.nio.charset.StandardCharsets.UTF_8), Map.class); + } + + @SuppressWarnings("unchecked") + @jakarta.annotation.Nullable + private static Map findParam( + @Nonnull final Map parameters, @Nonnull final String name) { + final List> list = (List>) parameters.get("parameter"); + if (list == null) { + return null; + } + return list.stream().filter(p -> name.equals(p.get("name"))).findFirst().orElse(null); + } + + @jakarta.annotation.Nullable + private static String findParamValue( + @Nonnull final Map parameters, + @Nonnull final String name, + @Nonnull final String valueKey) { + final Map param = findParam(parameters, name); + return param == null ? null : (String) param.get(valueKey); + } + + @SuppressWarnings("unchecked") + @jakarta.annotation.Nullable + private static Map findPart( + @Nonnull final Map param, @Nonnull final String partName) { + final List> parts = (List>) param.get("part"); + if (parts == null) { + return null; + } + return parts.stream().filter(p -> partName.equals(p.get("name"))).findFirst().orElse(null); + } + + @jakarta.annotation.Nullable + private static String findPartValue( + @Nonnull final Map param, + @Nonnull final String partName, + @Nonnull final String valueKey) { + final Map part = findPart(param, partName); + return part == null ? null : (String) part.get(valueKey); + } + /** * Creates a Parameters JSON with nested view structure matching what the UI sends. * diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java index ee24d7dc59..557d64761d 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderTest.java @@ -43,6 +43,7 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.Set; @@ -74,6 +75,7 @@ class ViewDefinitionExportProviderTest { @Mock private FhirContext fhirContext; @Mock private QueryableDataSource deltaLake; @Mock private GroupMemberService groupMemberService; + @Mock private ViewExecutionHelper viewExecutionHelper; @Mock private ServletRequestDetails requestDetails; private ViewDefinitionExportProvider provider; @@ -89,7 +91,8 @@ void setUp() { serverConfiguration, fhirContext, deltaLake, - groupMemberService); + groupMemberService, + viewExecutionHelper); } @AfterEach @@ -214,6 +217,7 @@ void exportReturnsResultFromAsyncContext() { when(job.getPreAsyncValidationResult()).thenReturn(exportRequest); when(job.isCancelled()).thenReturn(false); when(job.getId()).thenReturn("job-1"); + when(job.getStartTime()).thenReturn(Instant.parse("2026-06-21T01:00:00Z")); when(job.getOwnerId()).thenReturn(Optional.empty()); when(executor.execute(exportRequest, "job-1")).thenReturn(List.of()); lenient().when(serverConfiguration.getExport()).thenReturn(exportConfig); diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponseTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponseTest.java index 42c3469155..7a68c256d1 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponseTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponseTest.java @@ -19,9 +19,10 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.time.Instant; import java.util.List; -import org.hl7.fhir.r4.model.BooleanType; import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.IntegerType; import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; import org.hl7.fhir.r4.model.StringType; @@ -29,170 +30,121 @@ import org.junit.jupiter.api.Test; /** - * Unit tests for {@link ViewDefinitionExportResponse}. + * Unit tests for {@link ViewDefinitionExportResponse}, verifying the SQL on FHIR completion + * manifest shape (US9). * * @author John Grimes */ class ViewDefinitionExportResponseTest { + private static final String BASE_URL = "http://example.org/fhir"; + private static final Instant START = Instant.parse("2026-06-21T01:00:00Z"); + private static final Instant END = Instant.parse("2026-06-21T01:00:12Z"); + // ------------------------------------------------------------------------- - // Manifest structure tests + // Required manifest fields // ------------------------------------------------------------------------- @Test - void manifestContainsRequiredFields() { - // The manifest should contain transactionTime, request, requiresAccessToken, output, and - // error. - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(), - false); - - final Parameters parameters = response.toOutput(); + void manifestContainsExportIdAndCompletedStatus() { + final Parameters parameters = response("job-123", "tracking-1", "ndjson", List.of()).toOutput(); - assertThat(hasParameter(parameters, "transactionTime")).isTrue(); - assertThat(hasParameter(parameters, "request")).isTrue(); - assertThat(hasParameter(parameters, "requiresAccessToken")).isTrue(); - // Output and error may be absent when empty. + assertThat(getStringParameter(parameters, "exportId")).isEqualTo("job-123"); + assertThat(getStringParameter(parameters, "status")).isEqualTo("completed"); } @Test - void manifestContainsKickOffRequest() { - // The request parameter should contain the kick-off URL. - final String kickOffUrl = "http://example.org/fhir/$viewdefinition-export?_format=csv"; - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse(kickOffUrl, "http://example.org/fhir", List.of(), false); - - final Parameters parameters = response.toOutput(); - final String request = getStringParameter(parameters, "request"); - - assertThat(request).isEqualTo(kickOffUrl); + void manifestEchoesEffectiveFormat() { + final Parameters parameters = response("job-123", null, "csv", List.of()).toOutput(); + assertThat(getStringParameter(parameters, "_format")).isEqualTo("csv"); } @Test - void manifestShowsRequiresAccessTokenFalse() { - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(), - false); - - final Parameters parameters = response.toOutput(); - final Boolean requiresAccessToken = getBooleanParameter(parameters, "requiresAccessToken"); + void manifestEchoesClientTrackingIdWhenSupplied() { + final Parameters parameters = + response("job-123", "tracking-abc", "ndjson", List.of()).toOutput(); + assertThat(getStringParameter(parameters, "clientTrackingId")).isEqualTo("tracking-abc"); + } - assertThat(requiresAccessToken).isFalse(); + @Test + void manifestOmitsClientTrackingIdWhenAbsent() { + final Parameters parameters = response("job-123", null, "ndjson", List.of()).toOutput(); + assertThat(hasParameter(parameters, "clientTrackingId")).isFalse(); } @Test - void manifestShowsRequiresAccessTokenTrue() { - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(), - true); + void manifestContainsTimingFields() { + final Parameters parameters = response("job-123", null, "ndjson", List.of()).toOutput(); - final Parameters parameters = response.toOutput(); - final Boolean requiresAccessToken = getBooleanParameter(parameters, "requiresAccessToken"); + assertThat(findParameter(parameters, "exportStartTime").getValue()) + .isInstanceOf(InstantType.class); + assertThat(findParameter(parameters, "exportEndTime").getValue()) + .isInstanceOf(InstantType.class); - assertThat(requiresAccessToken).isTrue(); + final ParametersParameterComponent duration = findParameter(parameters, "exportDuration"); + assertThat(duration.getValue()).isInstanceOf(IntegerType.class); + assertThat(((IntegerType) duration.getValue()).getValue()).isEqualTo(12); } @Test - void manifestContainsTransactionTime() { - // The transactionTime should be an instant value. - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(), - false); + void manifestDoesNotContainNonSpecFields() { + final Parameters parameters = response("job-123", "t", "ndjson", List.of()).toOutput(); - final Parameters parameters = response.toOutput(); - final ParametersParameterComponent param = findParameter(parameters, "transactionTime"); - - assertThat(param).isNotNull(); - assertThat(param.getValue()).isInstanceOf(InstantType.class); + assertThat(hasParameter(parameters, "transactionTime")).isFalse(); + assertThat(hasParameter(parameters, "request")).isFalse(); + assertThat(hasParameter(parameters, "requiresAccessToken")).isFalse(); } // ------------------------------------------------------------------------- - // Output entries tests + // Output entries // ------------------------------------------------------------------------- @Test - void manifestContainsOutputEntries() { - // Output entries should have name and url parts. + void outputHasNameAndLocationPartsAndNoUrlPart() { final ViewExportOutput output = new ViewExportOutput( "patients", List.of("file:///tmp/jobs/abc-123/patients.ndjson/part-00000.json")); + final Parameters parameters = response("abc-123", null, "ndjson", List.of(output)).toOutput(); - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(output), - false); - - final Parameters parameters = response.toOutput(); final List outputs = getParametersByName(parameters, "output"); - assertThat(outputs).hasSize(1); final ParametersParameterComponent outputParam = outputs.get(0); assertThat(getPartValue(outputParam, "name")).isEqualTo("patients"); - // The URL should be transformed to a result URL. - final String url = getPartValue(outputParam, "url"); - assertThat(url).contains("$result").contains("job=abc-123"); + assertThat(getPartValue(outputParam, "location")).contains("$result").contains("job=abc-123"); + assertThat(hasPart(outputParam, "url")).isFalse(); } @Test - void manifestContainsMultipleFilesPerOutput() { - // When an output has multiple files, each gets its own output parameter. + void viewWithMultipleFilesYieldsOneOutputWithRepeatingLocations() { final ViewExportOutput output = new ViewExportOutput( "observations", List.of( "file:///tmp/jobs/job-id/observations.ndjson/part-00000.json", "file:///tmp/jobs/job-id/observations.ndjson/part-00001.json")); + final Parameters parameters = response("job-id", null, "ndjson", List.of(output)).toOutput(); - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(output), - false); - - final Parameters parameters = response.toOutput(); final List outputs = getParametersByName(parameters, "output"); - - assertThat(outputs).hasSize(2); - assertThat(getPartValue(outputs.get(0), "name")).isEqualTo("observations"); - assertThat(getPartValue(outputs.get(1), "name")).isEqualTo("observations"); + // One output per view, regardless of the number of files. + assertThat(outputs).hasSize(1); + final List locations = + outputs.get(0).getPart().stream().filter(p -> "location".equals(p.getName())).toList(); + assertThat(locations).hasSize(2); } @Test - void manifestContainsMultipleOutputs() { - // Multiple ViewExportOutputs each produce their own output parameters. - final ViewExportOutput output1 = + void multipleViewsYieldOneOutputEach() { + final ViewExportOutput patients = new ViewExportOutput( "patients", List.of("file:///tmp/jobs/job-id/patients.csv/part-00000.csv")); - final ViewExportOutput output2 = + final ViewExportOutput observations = new ViewExportOutput( "observations", List.of("file:///tmp/jobs/job-id/observations.csv/part-00000.csv")); + final Parameters parameters = + response("job-id", null, "csv", List.of(patients, observations)).toOutput(); - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(output1, output2), - false); - - final Parameters parameters = response.toOutput(); final List outputs = getParametersByName(parameters, "output"); - assertThat(outputs).hasSize(2); assertThat(getPartValue(outputs.get(0), "name")).isEqualTo("patients"); assertThat(getPartValue(outputs.get(1), "name")).isEqualTo("observations"); @@ -200,73 +152,39 @@ void manifestContainsMultipleOutputs() { @Test void noOutputParametersWhenNoOutputs() { - // When there are no outputs, no output parameters should be present. - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", - List.of(), - false); - - final Parameters parameters = response.toOutput(); - final List outputs = getParametersByName(parameters, "output"); - - assertThat(outputs).isEmpty(); - } - - // ------------------------------------------------------------------------- - // URL normalisation tests - // ------------------------------------------------------------------------- - - @Test - void serverBaseUrlNormalisedWithTrailingSlash() { - // Base URL without trailing slash should still produce valid URLs. - final ViewExportOutput output = - new ViewExportOutput( - "test", List.of("file:///tmp/jobs/job-id/test.ndjson/part-00000.json")); - - final ViewDefinitionExportResponse response = - new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir", // No trailing slash. - List.of(output), - false); - - final Parameters parameters = response.toOutput(); - final List outputs = getParametersByName(parameters, "output"); - final String url = getPartValue(outputs.get(0), "url"); - - // Should still produce a valid URL. - assertThat(url).startsWith("http://example.org/fhir/$result"); + final Parameters parameters = response("job-id", null, "ndjson", List.of()).toOutput(); + assertThat(getParametersByName(parameters, "output")).isEmpty(); } @Test - void serverBaseUrlWithTrailingSlashHandledCorrectly() { - // Base URL with trailing slash should not produce double slashes. + void serverBaseUrlNormalisationDoesNotProduceDoubleSlash() { final ViewExportOutput output = new ViewExportOutput( "test", List.of("file:///tmp/jobs/job-id/test.ndjson/part-00000.json")); - final ViewDefinitionExportResponse response = new ViewDefinitionExportResponse( - "http://example.org/fhir/$viewdefinition-export", - "http://example.org/fhir/", // With trailing slash. - List.of(output), - false); - - final Parameters parameters = response.toOutput(); - final List outputs = getParametersByName(parameters, "output"); - final String url = getPartValue(outputs.get(0), "url"); + BASE_URL + "/", List.of(output), "job-id", null, "ndjson", START, END); - // Should not have double slash. - assertThat(url).startsWith("http://example.org/fhir/$result"); - assertThat(url).doesNotContain("fhir//$result"); + final List outputs = + getParametersByName(response.toOutput(), "output"); + final String location = getPartValue(outputs.get(0), "location"); + assertThat(location).startsWith("http://example.org/fhir/$result"); + assertThat(location).doesNotContain("fhir//$result"); } // ------------------------------------------------------------------------- // Helper methods // ------------------------------------------------------------------------- + private static ViewDefinitionExportResponse response( + final String exportId, + final String clientTrackingId, + final String format, + final List outputs) { + return new ViewDefinitionExportResponse( + BASE_URL, outputs, exportId, clientTrackingId, format, START, END); + } + private static boolean hasParameter(final Parameters parameters, final String name) { return parameters.getParameter().stream().anyMatch(p -> name.equals(p.getName())); } @@ -289,24 +207,11 @@ private static String getStringParameter(final Parameters parameters, final Stri if (param == null || !param.hasValue()) { return null; } - if (param.getValue() instanceof final UriType uriType) { - return uriType.getValue(); - } - if (param.getValue() instanceof final StringType stringType) { - return stringType.getValue(); - } return param.getValue().primitiveValue(); } - private static Boolean getBooleanParameter(final Parameters parameters, final String name) { - final ParametersParameterComponent param = findParameter(parameters, name); - if (param == null || !param.hasValue()) { - return null; - } - if (param.getValue() instanceof final BooleanType booleanType) { - return booleanType.getValue(); - } - return null; + private static boolean hasPart(final ParametersParameterComponent param, final String partName) { + return param.getPart().stream().anyMatch(p -> partName.equals(p.getName())); } private static String getPartValue( From f5211486130663be2360e1d5048876faeab5c218 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:20:42 +1000 Subject: [PATCH 003/162] feat: Point SQL on FHIR conformance at spec canonicals; lock in sqlquery 422 Declare the SQL on FHIR spec canonical OperationDefinition URLs for the viewdefinition-run, viewdefinition-export, and sqlquery-run operations in the CapabilityStatement, and stop serving the Pathling-authored OperationDefinitions for them. Add a regression test pinning the existing 422 response for an unmappable FHIR column type under _format=fhir on sqlquery-run, and integration coverage for the source and _format rejections across all levels and transports. --- .../pathling/fhir/ConformanceProvider.java | 77 +++++-- .../fhir/run.OperationDefinition.json | 82 ------- .../sqlquery-run.OperationDefinition.json | 66 ------ ...definition-export.OperationDefinition.json | 90 -------- ...iewdefinition-run.OperationDefinition.json | 81 ------- .../fhir/ConformanceProviderTest.java | 81 +++++++ .../sqlquery/SqlQueryRunProviderIT.java | 216 ++++++++++++++++++ 7 files changed, 352 insertions(+), 341 deletions(-) delete mode 100644 server/src/main/resources/fhir/run.OperationDefinition.json delete mode 100644 server/src/main/resources/fhir/sqlquery-run.OperationDefinition.json delete mode 100644 server/src/main/resources/fhir/viewdefinition-export.OperationDefinition.json delete mode 100644 server/src/main/resources/fhir/viewdefinition-run.OperationDefinition.json diff --git a/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java b/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java index 0216fdb3b1..a13d2a4284 100644 --- a/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java +++ b/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java @@ -104,17 +104,31 @@ public class ConformanceProvider private static final String EXPORT_OPERATION = "export"; private static final String RUN_OPERATION = "run"; + private static final String VIEWDEFINITION_RUN_OPERATION = "viewdefinition-run"; + private static final String VIEWDEFINITION_EXPORT_OPERATION = "viewdefinition-export"; + private static final String SQLQUERY_RUN_OPERATION = "sqlquery-run"; - /** Base system-level operations available within Pathling. */ + /** + * The spec canonical OperationDefinition URLs for the SQL on FHIR operations. The server declares + * these in the CapabilityStatement instead of Pathling-authored OperationDefinitions, and no + * longer serves a private OperationDefinition for these operations. + */ + private static final String SOF_VIEWDEFINITION_RUN_CANONICAL = + "http://sql-on-fhir.org/OperationDefinition/$viewdefinition-run"; + + private static final String SOF_VIEWDEFINITION_EXPORT_CANONICAL = + "http://sql-on-fhir.org/OperationDefinition/$viewdefinition-export"; + + private static final String SOF_SQLQUERY_RUN_CANONICAL = + "http://sql-on-fhir.org/OperationDefinition/$sqlquery-run"; + + /** + * Base system-level operations whose Pathling-authored OperationDefinition resources are served. + * The SQL on FHIR run/export operations are intentionally excluded: they declare the spec + * canonical and Pathling does not serve an OperationDefinition for them. + */ private static final List BASE_SYSTEM_OPERATIONS = - Arrays.asList( - "job", - "result", - EXPORT_OPERATION, - "import", - "import-pnp", - "viewdefinition-run", - "viewdefinition-export"); + Arrays.asList("job", "result", EXPORT_OPERATION, "import", "import-pnp"); /** Bulk submit operations, added when bulk submit is configured. */ private static final List BULK_SUBMIT_OPERATIONS = @@ -129,9 +143,12 @@ public class ConformanceProvider private static final String FHIR_RESOURCE_BASE = "http://hl7.org/fhir/StructureDefinition/"; private static final String UNKNOWN_VERSION = "UNKNOWN"; - /** All resource-level operations available within Pathling. */ - private static final List RESOURCE_LEVEL_OPERATIONS = - List.of(EXPORT_OPERATION, RUN_OPERATION); + /** + * Resource-level operations whose Pathling-authored OperationDefinition resources are served. The + * ViewDefinition {@code $run} operation is excluded: it declares the spec canonical and Pathling + * does not serve an OperationDefinition for it. + */ + private static final List RESOURCE_LEVEL_OPERATIONS = List.of(EXPORT_OPERATION); /** Resource types that have the export operation available. */ private static final Set EXPORT_RESOURCE_TYPES = @@ -468,9 +485,9 @@ private List buildResources() { viewDefResource.addInteraction(viewDefDeleteInteraction); } - // Add $run operation to ViewDefinition resource if enabled. + // Add $run operation to ViewDefinition resource if enabled, declaring the spec canonical. if (ops.isViewDefinitionInstanceRunEnabled()) { - final CanonicalType runUri = new CanonicalType(getOperationUri(RUN_OPERATION)); + final CanonicalType runUri = new CanonicalType(SOF_VIEWDEFINITION_RUN_CANONICAL); final CapabilityStatementRestResourceOperationComponent runOp = new CapabilityStatementRestResourceOperationComponent( new StringType(RUN_OPERATION), runUri); @@ -500,12 +517,21 @@ private List buildOperations( addOperationIfEnabled(operations, "import", ops.isImportEnabled()); addOperationIfEnabled(operations, "import-pnp", ops.isImportPnpEnabled()); - // Add viewdefinition operations. - addOperationIfEnabled(operations, "viewdefinition-run", ops.isViewDefinitionRunEnabled()); - addOperationIfEnabled(operations, "viewdefinition-export", ops.isViewDefinitionExportEnabled()); - - // Add SQL query run operation. - addOperationIfEnabled(operations, "sqlquery-run", ops.isSqlQueryRunEnabled()); + // Add viewdefinition operations, declaring the SQL on FHIR spec canonicals. + addOperationIfEnabled( + operations, + VIEWDEFINITION_RUN_OPERATION, + ops.isViewDefinitionRunEnabled(), + SOF_VIEWDEFINITION_RUN_CANONICAL); + addOperationIfEnabled( + operations, + VIEWDEFINITION_EXPORT_OPERATION, + ops.isViewDefinitionExportEnabled(), + SOF_VIEWDEFINITION_EXPORT_CANONICAL); + + // Add SQL query run operation, declaring the SQL on FHIR spec canonical. + addOperationIfEnabled( + operations, SQLQUERY_RUN_OPERATION, ops.isSqlQueryRunEnabled(), SOF_SQLQUERY_RUN_CANONICAL); // Add bulk submit operations if configured and enabled. if (configuration.getBulkSubmit() != null && ops.isBulkSubmitEnabled()) { @@ -520,11 +546,18 @@ private void addOperationIfEnabled( final List operations, final String name, final boolean enabled) { + addOperationIfEnabled(operations, name, enabled, getOperationUri(name)); + } + + private void addOperationIfEnabled( + final List operations, + final String name, + final boolean enabled, + final String definitionUri) { if (enabled) { - final CanonicalType operationUri = new CanonicalType(getOperationUri(name)); operations.add( new CapabilityStatementRestResourceOperationComponent( - new StringType(name), operationUri)); + new StringType(name), new CanonicalType(definitionUri))); } } diff --git a/server/src/main/resources/fhir/run.OperationDefinition.json b/server/src/main/resources/fhir/run.OperationDefinition.json deleted file mode 100644 index 5c131a8e92..0000000000 --- a/server/src/main/resources/fhir/run.OperationDefinition.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "resourceType": "OperationDefinition", - "name": "run", - "title": "Pathling ViewDefinition Run Operation (Type/Instance Level)", - "status": "active", - "kind": "operation", - "experimental": false, - "publisher": "Australian e-Health Research Centre, CSIRO", - "description": "This operation executes a SQL on FHIR ViewDefinition against the data within the Pathling server, returning the result as a streaming NDJSON or CSV response. When invoked at the type level (/ViewDefinition/$run), a viewResource parameter must be provided. When invoked at the instance level (/ViewDefinition/[id]/$run), the stored ViewDefinition is used.", - "affectsState": false, - "code": "run", - "system": false, - "type": true, - "instance": true, - "resource": ["Basic"], - "parameter": [ - { - "name": "viewResource", - "use": "in", - "min": 0, - "max": "1", - "documentation": "A ViewDefinition resource that defines the view to execute. Required for type-level invocation, ignored for instance-level invocation where the stored ViewDefinition is used.", - "type": "Resource" - }, - { - "name": "_format", - "use": "in", - "min": 0, - "max": "1", - "documentation": "The output format for the view results. Supported values are 'application/x-ndjson' (default) and 'text/csv'.", - "type": "code" - }, - { - "name": "header", - "use": "in", - "min": 0, - "max": "1", - "documentation": "For CSV output, whether to include a header row. Defaults to true.", - "type": "boolean" - }, - { - "name": "patient", - "use": "in", - "min": 0, - "max": "1", - "documentation": "A patient ID to filter the view results to resources within that patient's compartment.", - "type": "id" - }, - { - "name": "group", - "use": "in", - "min": 0, - "max": "1", - "documentation": "A group ID to filter the view results to resources within the compartments of patients in that group.", - "type": "id" - }, - { - "name": "_since", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Only include resources that have been updated since the specified timestamp.", - "type": "instant" - }, - { - "name": "_limit", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Maximum number of rows to return in the result.", - "type": "integer" - }, - { - "name": "resource", - "use": "in", - "min": 0, - "max": "*", - "documentation": "Inline FHIR resources to use as the data source instead of the server's main data. When provided, the view is executed against these resources rather than the persisted data.", - "type": "Resource" - } - ] -} diff --git a/server/src/main/resources/fhir/sqlquery-run.OperationDefinition.json b/server/src/main/resources/fhir/sqlquery-run.OperationDefinition.json deleted file mode 100644 index 3b35d220b1..0000000000 --- a/server/src/main/resources/fhir/sqlquery-run.OperationDefinition.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "resourceType": "OperationDefinition", - "name": "sqlquery-run", - "title": "Pathling SQLQuery Run Operation", - "status": "active", - "kind": "operation", - "experimental": false, - "publisher": "Australian e-Health Research Centre, CSIRO", - "description": "This operation executes a SQL query against materialised ViewDefinition tables within the Pathling server. The query is provided as a Library resource conforming to the SQLQuery profile from the SQL on FHIR v2 specification.", - "affectsState": false, - "code": "sqlquery-run", - "system": true, - "type": true, - "instance": true, - "resource": ["Library"], - "parameter": [ - { - "name": "queryResource", - "use": "in", - "min": 0, - "max": "1", - "documentation": "An inline Library resource conforming to the SQLQuery profile. Contains the SQL query, ViewDefinition dependencies, and parameter declarations.", - "type": "Resource" - }, - { - "name": "queryReference", - "use": "in", - "min": 0, - "max": "1", - "documentation": "A reference to a stored Library resource conforming to the SQLQuery profile.", - "type": "Reference" - }, - { - "name": "_format", - "use": "in", - "min": 0, - "max": "1", - "documentation": "The output format for the query results. Supported values are 'ndjson' (default), 'csv', 'json', 'parquet', and 'fhir'. The 'fhir' format returns a Parameters resource (application/fhir+json) with one repeating 'row' parameter per result row, each column rendered as a part with a typed value[x].", - "type": "code" - }, - { - "name": "header", - "use": "in", - "min": 0, - "max": "1", - "documentation": "For CSV output, whether to include a header row. Defaults to true.", - "type": "boolean" - }, - { - "name": "_limit", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Maximum number of rows to return in the result.", - "type": "integer" - }, - { - "name": "parameters", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Runtime parameter bindings as a Parameters resource. Each entry's name must match a Library.parameter declaration, and its value[x] must match the declared FHIR type.", - "type": "Parameters" - } - ] -} diff --git a/server/src/main/resources/fhir/viewdefinition-export.OperationDefinition.json b/server/src/main/resources/fhir/viewdefinition-export.OperationDefinition.json deleted file mode 100644 index e4592cf815..0000000000 --- a/server/src/main/resources/fhir/viewdefinition-export.OperationDefinition.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "resourceType": "OperationDefinition", - "name": "viewdefinition-export", - "title": "Pathling ViewDefinition Export Operation", - "status": "active", - "kind": "operation", - "experimental": false, - "publisher": "Australian e-Health Research Centre, CSIRO", - "description": "This operation executes one or more SQL on FHIR ViewDefinitions against the data within the Pathling server, writing the results to files that can be downloaded asynchronously. Supports NDJSON, CSV, and Parquet output formats.", - "affectsState": false, - "code": "viewdefinition-export", - "system": true, - "type": false, - "instance": false, - "parameter": [ - { - "name": "view", - "use": "in", - "min": 1, - "max": "*", - "documentation": "One or more ViewDefinitions to export. Each view produces a separate output file.", - "part": [ - { - "name": "name", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Optional name for the output. If not provided, the ViewDefinition's resource type will be used with an index suffix.", - "type": "string" - }, - { - "name": "viewResource", - "use": "in", - "min": 1, - "max": "1", - "documentation": "The ViewDefinition resource defining the view to execute.", - "type": "Resource" - } - ] - }, - { - "name": "clientTrackingId", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Client-provided tracking identifier for the export request.", - "type": "string" - }, - { - "name": "_format", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Output format for the exported files. Supported values: 'ndjson' (default), 'csv', 'parquet'.", - "type": "code" - }, - { - "name": "header", - "use": "in", - "min": 0, - "max": "1", - "documentation": "For CSV output, whether to include a header row. Defaults to true.", - "type": "boolean" - }, - { - "name": "patient", - "use": "in", - "min": 0, - "max": "*", - "documentation": "Patient ID(s) to filter the view results to resources within those patients' compartments.", - "type": "id" - }, - { - "name": "group", - "use": "in", - "min": 0, - "max": "*", - "documentation": "Group ID(s) to filter the view results to resources within the compartments of patients in those groups.", - "type": "id" - }, - { - "name": "_since", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Only include resources that have been updated since the specified timestamp.", - "type": "instant" - } - ] -} diff --git a/server/src/main/resources/fhir/viewdefinition-run.OperationDefinition.json b/server/src/main/resources/fhir/viewdefinition-run.OperationDefinition.json deleted file mode 100644 index bae0a8a849..0000000000 --- a/server/src/main/resources/fhir/viewdefinition-run.OperationDefinition.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "resourceType": "OperationDefinition", - "name": "viewdefinition-run", - "title": "Pathling ViewDefinition Run Operation", - "status": "active", - "kind": "operation", - "experimental": false, - "publisher": "Australian e-Health Research Centre, CSIRO", - "description": "This operation executes a SQL on FHIR ViewDefinition against the data within the Pathling server, returning the result as a streaming NDJSON or CSV response.", - "affectsState": false, - "code": "viewdefinition-run", - "system": true, - "type": false, - "instance": false, - "parameter": [ - { - "name": "viewResource", - "use": "in", - "min": 1, - "max": "1", - "documentation": "A ViewDefinition resource that defines the view to execute.", - "type": "Resource" - }, - { - "name": "_format", - "use": "in", - "min": 0, - "max": "1", - "documentation": "The output format for the view results. Supported values are 'application/x-ndjson' (default) and 'text/csv'.", - "type": "code" - }, - { - "name": "header", - "use": "in", - "min": 0, - "max": "1", - "documentation": "For CSV output, whether to include a header row. Defaults to true.", - "type": "boolean" - }, - { - "name": "patient", - "use": "in", - "min": 0, - "max": "1", - "documentation": "A patient ID to filter the view results to resources within that patient's compartment.", - "type": "id" - }, - { - "name": "group", - "use": "in", - "min": 0, - "max": "1", - "documentation": "A group ID to filter the view results to resources within the compartments of patients in that group.", - "type": "id" - }, - { - "name": "_since", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Only include resources that have been updated since the specified timestamp.", - "type": "instant" - }, - { - "name": "_limit", - "use": "in", - "min": 0, - "max": "1", - "documentation": "Maximum number of rows to return in the result.", - "type": "integer" - }, - { - "name": "resource", - "use": "in", - "min": 0, - "max": "*", - "documentation": "Inline FHIR resources to use as the data source instead of the server's main data. When provided, the view is executed against these resources rather than the persisted data.", - "type": "Resource" - } - ] -} diff --git a/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java b/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java index d25cbdd690..7a7ecefa57 100644 --- a/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java @@ -18,6 +18,7 @@ package au.csiro.pathling.fhir; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import au.csiro.pathling.FhirServer; import au.csiro.pathling.PathlingServerVersion; @@ -25,6 +26,7 @@ import au.csiro.pathling.config.OperationConfiguration; import au.csiro.pathling.config.ServerConfiguration; import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.errors.ResourceNotFoundError; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirVersionEnum; import ca.uhn.fhir.parser.IParser; @@ -42,6 +44,7 @@ import org.hl7.fhir.r4.model.CapabilityStatement.TypeRestfulInteraction; import org.hl7.fhir.r4.model.Enumerations.ResourceType; import org.hl7.fhir.r4.model.Enumerations.SearchParamType; +import org.hl7.fhir.r4.model.IdType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -234,6 +237,62 @@ void capabilityStatementIncludesViewDefinitionExportOperation() { .contains("viewdefinition-export"); } + // ------------------------------------------------------------------------- + // SQL on FHIR conformance canonicals (US2) + // ------------------------------------------------------------------------- + + @Test + void viewDefinitionRunOperationsDeclareSpecCanonical() { + final CapabilityStatement capabilityStatement = + conformanceProvider.getServerConformance(null, null); + + // The system-level viewdefinition-run operation declares the run canonical. + assertThat(systemOperationDefinition(capabilityStatement, "viewdefinition-run")) + .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$viewdefinition-run"); + + // The ViewDefinition-level run operation declares the same run canonical. + assertThat(resourceOperationDefinition(capabilityStatement, "ViewDefinition", "run")) + .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$viewdefinition-run"); + } + + @Test + void viewDefinitionExportDeclaresSpecCanonical() { + final CapabilityStatement capabilityStatement = + conformanceProvider.getServerConformance(null, null); + assertThat(systemOperationDefinition(capabilityStatement, "viewdefinition-export")) + .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$viewdefinition-export"); + } + + @Test + void sqlQueryRunDeclaresSpecCanonical() { + final CapabilityStatement capabilityStatement = + conformanceProvider.getServerConformance(null, null); + assertThat(systemOperationDefinition(capabilityStatement, "sqlquery-run")) + .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$sqlquery-run"); + } + + @Test + void authoredSqlOnFhirOperationDefinitionsNoLongerServed() { + for (final String name : + List.of("run", "viewdefinition-run", "viewdefinition-export", "sqlquery-run")) { + assertThatThrownBy( + () -> + conformanceProvider.getOperationDefinitionById( + new IdType("OperationDefinition/" + name + "-1"))) + .as("OperationDefinition for %s should no longer be served", name) + .isInstanceOf(ResourceNotFoundError.class); + } + } + + @Test + void otherOperationDefinitionsStillServed() { + // The Bulk Data export OperationDefinition continues to be served unchanged. + assertThat( + conformanceProvider.getOperationDefinitionById( + new IdType("OperationDefinition/export-1"))) + .isNotNull(); + } + @Test void capabilityStatementHasNoDuplicateResourceTypes() { // When: Getting the capability statement. @@ -576,6 +635,28 @@ private CapabilityStatementRestResourceComponent findResource( .orElseThrow(() -> new AssertionError("Resource not found: " + typeCode)); } + /** Returns the declared {@code definition} canonical for a system-level operation, or null. */ + private String systemOperationDefinition( + final CapabilityStatement capabilityStatement, final String operationName) { + return capabilityStatement.getRest().getFirst().getOperation().stream() + .filter(o -> operationName.equals(o.getName())) + .map(CapabilityStatementRestResourceOperationComponent::getDefinition) + .findFirst() + .orElse(null); + } + + /** Returns the declared {@code definition} canonical for a resource-level operation, or null. */ + private String resourceOperationDefinition( + final CapabilityStatement capabilityStatement, + final String typeCode, + final String operationName) { + return findResource(capabilityStatement, typeCode).getOperation().stream() + .filter(o -> operationName.equals(o.getName())) + .map(CapabilityStatementRestResourceOperationComponent::getDefinition) + .findFirst() + .orElse(null); + } + /** * Helper method to create a ConformanceProvider with custom operation configuration. * diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java index d27b7287bb..db61f3d0fb 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java @@ -198,6 +198,222 @@ void instanceLevelRunWithNonExistentLibraryReturns404() { .isNotFound(); } + // ------------------------------------------------------------------------- + // source rejection (US1, FR-001) — every level, over POST and GET + // ------------------------------------------------------------------------- + + @Test + void sourceRejectedAtSystemLevelOverPost() { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final Map queryResourceParam = new LinkedHashMap<>(); + queryResourceParam.put("name", "queryResource"); + queryResourceParam.put( + "resource", + GSON.fromJson( + jsonParser.encodeResourceToString(sqlQueryLibrary(SELF_CONTAINED_SQL)), Map.class)); + final Map sourceParam = new LinkedHashMap<>(); + sourceParam.put("name", "source"); + sourceParam.put("valueString", "external"); + parameters.put("parameter", List.of(queryResourceParam, sourceParam)); + + sourceRejected("/fhir/$sqlquery-run", GSON.toJson(parameters)); + } + + @Test + void sourceRejectedAtSystemLevelOverGet() { + getExpectStatus("/fhir/$sqlquery-run?source=external&_format=ndjson", 400); + } + + @Test + void sourceRejectedAtTypeLevelOverGet() { + getExpectStatus("/fhir/Library/$sqlquery-run?source=external&_format=ndjson", 400); + } + + @Test + void sourceRejectedAtInstanceLevelOverGet() { + // The source rejection precedes the stored-Library read, so it applies even for a missing id. + getExpectStatus("/fhir/Library/some-id/$sqlquery-run?source=external&_format=ndjson", 400); + } + + /** Posts a body carrying a source part and asserts a 400 naming source. */ + private void sourceRejected(@Nonnull final String path, @Nonnull final String body) { + final byte[] content = + webTestClient + .post() + .uri("http://localhost:" + port + path) + .header("Content-Type", "application/fhir+json") + .bodyValue(body) + .exchange() + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + assertThat(new String(Objects.requireNonNull(content), StandardCharsets.UTF_8)) + .contains("source"); + } + + // ------------------------------------------------------------------------- + // _format handling (US1, FR-002/FR-003/FR-004) + // ------------------------------------------------------------------------- + + @Test + void unsupportedExplicitFormatReturns400() { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final Map queryResourceParam = new LinkedHashMap<>(); + queryResourceParam.put("name", "queryResource"); + queryResourceParam.put( + "resource", + GSON.fromJson( + jsonParser.encodeResourceToString(sqlQueryLibrary(SELF_CONTAINED_SQL)), Map.class)); + final Map formatParam = new LinkedHashMap<>(); + formatParam.put("name", "_format"); + formatParam.put("valueString", "xml"); + parameters.put("parameter", List.of(queryResourceParam, formatParam)); + + final byte[] content = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$sqlquery-run") + .header("Content-Type", "application/fhir+json") + .bodyValue(GSON.toJson(parameters)) + .exchange() + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + assertThat(new String(Objects.requireNonNull(content), StandardCharsets.UTF_8)).contains("xml"); + } + + @Test + void acceptHeaderMismatchDefaultsToNdjson() { + // No _format parameter and an Accept that matches no supported media type: defaults to NDJSON + // (the explicit-format rejection applies only to _format, not to content negotiation). + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final Map queryResourceParam = new LinkedHashMap<>(); + queryResourceParam.put("name", "queryResource"); + queryResourceParam.put( + "resource", + GSON.fromJson( + jsonParser.encodeResourceToString(sqlQueryLibrary(SELF_CONTAINED_SQL)), Map.class)); + parameters.put("parameter", List.of(queryResourceParam)); + + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$sqlquery-run") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/xml") + .bodyValue(GSON.toJson(parameters)) + .exchange() + .expectStatus() + .isOk() + .expectHeader() + .contentTypeCompatibleWith(MediaType.parseMediaType("application/x-ndjson")); + } + + // ------------------------------------------------------------------------- + // Conformance canonical (US2, FR-005/FR-006) + // ------------------------------------------------------------------------- + + @Test + @SuppressWarnings("unchecked") + void sqlQueryRunDeclaresSpecCanonicalAndForkNotServed() { + final byte[] metadata = + webTestClient + .get() + .uri("http://localhost:" + port + "/fhir/metadata") + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult() + .getResponseBodyContent(); + + final Map capability = + GSON.fromJson( + new String(Objects.requireNonNull(metadata), StandardCharsets.UTF_8), Map.class); + final List> rest = (List>) capability.get("rest"); + final List> operations = + (List>) rest.get(0).get("operation"); + + final String sqlQueryRunDefinition = + operations.stream() + .filter(o -> "sqlquery-run".equals(o.get("name"))) + .map(o -> (String) o.get("definition")) + .findFirst() + .orElse(null); + assertThat(sqlQueryRunDefinition) + .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$sqlquery-run"); + + // The Pathling-authored sqlquery-run OperationDefinition is no longer served. + getExpectStatus("/fhir/OperationDefinition/sqlquery-run", 404); + } + + // ------------------------------------------------------------------------- + // 422 for an unmappable FHIR column type (US11, FR-036/FR-037) + // ------------------------------------------------------------------------- + + @Test + void fhirFormatWithUnmappableColumnTypeReturns422() { + // _format=fhir requires every column to map to a FHIR value; an array column cannot, so the + // shared streaming helper rejects it up front with 422 before any bytes are written. This + // regression test guards the existing behaviour in ResultStreamingHelper. + final String arrayColumnSql = "SELECT 1 AS id, array(1, 2, 3) AS arr"; + + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final Map queryResourceParam = new LinkedHashMap<>(); + queryResourceParam.put("name", "queryResource"); + queryResourceParam.put( + "resource", + GSON.fromJson( + jsonParser.encodeResourceToString(sqlQueryLibrary(arrayColumnSql)), Map.class)); + final Map formatParam = new LinkedHashMap<>(); + formatParam.put("name", "_format"); + formatParam.put("valueString", "fhir"); + parameters.put("parameter", List.of(queryResourceParam, formatParam)); + final String body = GSON.toJson(parameters); + + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$sqlquery-run") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/fhir+json") + .bodyValue(body) + .exchange() + .expectStatus() + .isEqualTo(422); + + // The same query in a flat format (NDJSON) succeeds. + final Map ndjsonParameters = new LinkedHashMap<>(); + ndjsonParameters.put("resourceType", "Parameters"); + final Map ndjsonQueryParam = new LinkedHashMap<>(); + ndjsonQueryParam.put("name", "queryResource"); + ndjsonQueryParam.put( + "resource", + GSON.fromJson( + jsonParser.encodeResourceToString(sqlQueryLibrary(arrayColumnSql)), Map.class)); + ndjsonParameters.put("parameter", List.of(ndjsonQueryParam)); + + postOk("/fhir/$sqlquery-run", GSON.toJson(ndjsonParameters), SqlQueryOutputFormat.NDJSON); + } + + /** Issues a GET and asserts the given status code. */ + private void getExpectStatus(@Nonnull final String path, final int status) { + webTestClient + .get() + .uri("http://localhost:" + port + path) + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isEqualTo(status); + } + @Nonnull private String postOk( @Nonnull final String path, From 1d94664d0a5349733fa7cd6f7dd6eb8a29f29ef8 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:23:00 +1000 Subject: [PATCH 004/162] test: Confirm Bulk Data export kick-off body is unaffected by SoF change --- .../bulkexport/ExportOperationIT.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/src/test/java/au/csiro/pathling/operations/bulkexport/ExportOperationIT.java b/server/src/test/java/au/csiro/pathling/operations/bulkexport/ExportOperationIT.java index c075d43ac8..833f0af186 100644 --- a/server/src/test/java/au/csiro/pathling/operations/bulkexport/ExportOperationIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/bulkexport/ExportOperationIT.java @@ -225,6 +225,28 @@ void testInvalidKickoffRequest() { .isBadRequest(); } + @Test + void bulkExportKickOffBodyRemainsOperationOutcome() { + // The SQL on FHIR kick-off body change (a Parameters acknowledgement) applies only to the + // redirect-on-complete operations; the FHIR Bulk Data $export kick-off body is unchanged and + // remains an OperationOutcome. + final String uri = + "http://localhost:" + + port + + "/fhir/$export?_outputFormat=application/fhir+ndjson&_type=Patient"; + webTestClient + .get() + .uri(uri) + .header("Accept", "application/fhir+json") + .header("Prefer", "respond-async") + .exchange() + .expectStatus() + .isAccepted() + .expectBody() + .jsonPath("$.resourceType") + .isEqualTo("OperationOutcome"); + } + @Test void testExportValid() { final String uri = From 47e147bf4209a46a3466764a4f471ca886a08f46 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:29:57 +1000 Subject: [PATCH 005/162] feat: Read view-export manifest download files from location parts Update the admin UI manifest parser to read the spec-shaped output location parts (one or more per output) instead of a single url part, emitting one download entry per file so a view that produced several files lists every file. Add an end-to-end check that an invalid custom ViewDefinition surfaces the 422 OperationOutcome message. --- ui/e2e/sqlOnFhir.spec.ts | 40 +++++++++++++++++ ui/src/types/__tests__/viewExport.test.ts | 53 +++++++++++++++++++---- ui/src/types/viewExport.ts | 13 ++++-- 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/ui/e2e/sqlOnFhir.spec.ts b/ui/e2e/sqlOnFhir.spec.ts index d8f5963d4f..065a332af2 100644 --- a/ui/e2e/sqlOnFhir.spec.ts +++ b/ui/e2e/sqlOnFhir.spec.ts @@ -462,6 +462,46 @@ test.describe("SQL on FHIR page", () => { // Verify error message is displayed. await expect(page.getByText(/View run failed/)).toBeVisible(); }); + + test("displays the 422 message for an invalid custom view definition", async ({ + page, + }) => { + // An invalid custom ViewDefinition now returns 422; the OperationOutcome message must be + // surfaced clearly rather than as a generic failure (US3). + const errorOutcome = { + resourceType: "OperationOutcome", + issue: [ + { + severity: "error", + diagnostics: "Invalid ViewDefinition: bad path", + }, + ], + }; + + await mockMetadata(page); + await mockViewDefinitions(page); + await mockViewRun(page, { status: 422, body: errorOutcome }); + + await page.goto("/admin/sql-on-fhir"); + + // Switch to the custom JSON tab and provide a view definition. + await page.getByRole("tab", { name: "Provide JSON" }).click(); + const jsonInput = page.locator("textarea").last(); + await jsonInput.fill( + JSON.stringify({ + resourceType: "ViewDefinition", + name: "Invalid View", + resource: "Patient", + status: "active", + select: [{ column: [{ path: "bad.path", name: "x" }] }], + }), + ); + + await page.getByRole("button", { name: "Execute" }).click(); + + // The OperationOutcome diagnostics are shown clearly. + await expect(page.getByText(/Invalid ViewDefinition/)).toBeVisible(); + }); }); test.describe("Export", () => { diff --git a/ui/src/types/__tests__/viewExport.test.ts b/ui/src/types/__tests__/viewExport.test.ts index 5af328e484..094651ef6d 100644 --- a/ui/src/types/__tests__/viewExport.test.ts +++ b/ui/src/types/__tests__/viewExport.test.ts @@ -28,18 +28,20 @@ import { getViewExportOutputFiles } from "../viewExport"; import type { Parameters } from "fhir/r4"; describe("getViewExportOutputFiles", () => { - it("extracts output files from Parameters resource", () => { - // A manifest with two output files. + it("extracts one file per output from a spec-shaped manifest", () => { + // A spec-shaped manifest where each output has a name and a single location part. const manifest: Parameters = { resourceType: "Parameters", parameter: [ - { name: "transactionTime", valueInstant: "2025-01-01T00:00:00Z" }, + { name: "exportId", valueString: "job-abc" }, + { name: "status", valueCode: "completed" }, + { name: "_format", valueCode: "ndjson" }, { name: "output", part: [ { name: "name", valueString: "patients" }, { - name: "url", + name: "location", valueUri: "http://example.org/$result?job=abc&file=patients.ndjson", }, @@ -50,7 +52,7 @@ describe("getViewExportOutputFiles", () => { part: [ { name: "name", valueString: "observations" }, { - name: "url", + name: "location", valueUri: "http://example.org/$result?job=abc&file=observations.ndjson", }, @@ -72,6 +74,43 @@ describe("getViewExportOutputFiles", () => { }); }); + it("emits one entry per location when a view produced multiple files", () => { + // A single output with several location parts (e.g. a Spark-partitioned view). + const manifest: Parameters = { + resourceType: "Parameters", + parameter: [ + { + name: "output", + part: [ + { name: "name", valueString: "observations" }, + { + name: "location", + valueUri: + "http://example.org/$result?job=abc&file=observations-0.ndjson", + }, + { + name: "location", + valueUri: + "http://example.org/$result?job=abc&file=observations-1.ndjson", + }, + ], + }, + ], + }; + + const outputs = getViewExportOutputFiles(manifest); + + expect(outputs).toHaveLength(2); + expect(outputs[0]).toEqual({ + name: "observations", + url: "http://example.org/$result?job=abc&file=observations-0.ndjson", + }); + expect(outputs[1]).toEqual({ + name: "observations", + url: "http://example.org/$result?job=abc&file=observations-1.ndjson", + }); + }); + it("returns empty array when no parameter property", () => { // A manifest with no parameters. const manifest: Parameters = { resourceType: "Parameters" }; @@ -82,9 +121,7 @@ describe("getViewExportOutputFiles", () => { // A manifest with parameters but no output entries. const manifest: Parameters = { resourceType: "Parameters", - parameter: [ - { name: "transactionTime", valueInstant: "2025-01-01T00:00:00Z" }, - ], + parameter: [{ name: "status", valueCode: "completed" }], }; expect(getViewExportOutputFiles(manifest)).toEqual([]); }); diff --git a/ui/src/types/viewExport.ts b/ui/src/types/viewExport.ts index 4b2d5f14a3..2a1472f686 100644 --- a/ui/src/types/viewExport.ts +++ b/ui/src/types/viewExport.ts @@ -44,8 +44,12 @@ export type ViewExportManifest = Parameters; /** * Extracts output file entries from a view export manifest Parameters resource. * + * Each SQL on FHIR `output` parameter carries one `name` part and one or more `location` parts + * (one per file the view produced). This emits one download entry per `location`, all sharing the + * output's name, so a view that produced several files lists every file. + * * @param manifest - The view export manifest Parameters resource. - * @returns Array of output file entries with name and url. + * @returns Array of output file entries with name and url, one per file. */ export function getViewExportOutputFiles( manifest: ViewExportManifest, @@ -56,10 +60,11 @@ export function getViewExportOutputFiles( return manifest.parameter .filter((param) => param.name === "output" && param.part) - .map((param) => { + .flatMap((param) => { const parts = param.part!; const name = parts.find((p) => p.name === "name")?.valueString ?? ""; - const url = parts.find((p) => p.name === "url")?.valueUri ?? ""; - return { name, url }; + return parts + .filter((p) => p.name === "location" && p.valueUri) + .map((p) => ({ name, url: p.valueUri! })); }); } From 5f3955052eecd3036a729652ea2699d56dd10e26 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:35:26 +1000 Subject: [PATCH 006/162] docs: Document SQL on FHIR operation alignment changes Update the run, export, and sqlquery-run operation docs for the new viewReference support, Reference-typed patient/group, Bundle unwrapping, the spec-shaped export manifest and kick-off acknowledgement, the source and unsupported-format rejections, the 400/404/422 status semantics, and the spec conformance canonicals. Credit the additional author on the significantly changed sqlquery providers. --- .../sqlquery/SqlQueryExecutionHelper.java | 2 + .../sqlquery/SqlQueryInstanceRunProvider.java | 1 + .../sqlquery/SqlQueryOutputFormat.java | 2 + .../sqlquery/SqlQueryRequestParser.java | 2 + .../sqlquery/SqlQueryRunProvider.java | 1 + site/docs/server/operations/sql-run.md | 42 ++++- site/docs/server/operations/view-export.md | 164 ++++++++++++------ site/docs/server/operations/view-run.md | 88 ++++++++-- 8 files changed, 221 insertions(+), 81 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java index 3cfbdb41a0..50d1a1e4d6 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java @@ -35,6 +35,8 @@ /** * Orchestrates the {@code $sqlquery-run} operation by chaining the parser, view resolver, executor * and result streamer. + * + * @author John Grimes */ @Component public class SqlQueryExecutionHelper { diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java index b9570e3af9..8fc1344f0a 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceRunProvider.java @@ -49,6 +49,7 @@ * Library * * + * @author John Grimes * @see SQLQueryRun * @see SqlQueryRunProvider for system-level $sqlquery-run operation diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java index cc70fb7ba5..c5f91cca4b 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java @@ -29,6 +29,8 @@ * Output format options for the {@code $sqlquery-run} operation. Supports NDJSON, CSV, JSON, * Parquet, and FHIR ({@code Parameters} resource) formats as specified in the SQL on FHIR v2 * specification. + * + * @author John Grimes */ @Getter public enum SqlQueryOutputFormat { diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java index 6b077fcc0f..202f2856bc 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java @@ -46,6 +46,8 @@ * Validates and normalises the raw HTTP inputs of a {@code $sqlquery-run} invocation into a {@link * SqlQueryRequest}. Has no Spark dependency; performs only structural FHIR-level validation and * parsing. + * + * @author John Grimes */ @Slf4j @Component diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java index 027b7942fe..b37c2db6ae 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProvider.java @@ -39,6 +39,7 @@ *

    This provides a system-level operation at {@code /fhir/$sqlquery-run} that accepts a SQLQuery * Library resource inline or by reference. * + * @author John Grimes * @see SQLQueryRun * @see SqlQueryInstanceRunProvider for type-level and instance-level operations diff --git a/site/docs/server/operations/sql-run.md b/site/docs/server/operations/sql-run.md index 931a9d5d6a..0711ce334b 100644 --- a/site/docs/server/operations/sql-run.md +++ b/site/docs/server/operations/sql-run.md @@ -26,19 +26,40 @@ type-level forms accept the Library inline or by reference. ## Parameters -| Name | Cardinality | Type | Description | -| ---------------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `queryResource` | 0..1 | Resource | An inline Library resource conforming to the SQLQuery profile. Mutually exclusive with `queryReference`. | -| `queryReference` | 0..1 | Reference | A relative literal (`Library/[id]`) or canonical (`[url]` or `[url]\|[version]`) reference to a stored Library. Resolves against the server's Library store. | -| `_format` | 0..1 | code | Output format. Accepts `ndjson` (default), `csv`, `json`, `parquet`, or `fhir`. | -| `header` | 0..1 | boolean | Include the header row in CSV output. Defaults to `true`. | -| `_limit` | 0..1 | integer | Maximum number of rows to return. Always clamped to the server-configured `maxRows` ceiling. | -| `parameters` | 0..1 | Parameters | Runtime parameter bindings. Each entry's name must match a `Library.parameter` declaration, and its `value[x]` must match the declared FHIR type. | +| Name | Cardinality | Type | Description | +| ---------------- | ----------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `queryResource` | 0..1 | Resource | An inline Library resource conforming to the SQLQuery profile. Mutually exclusive with `queryReference`. | +| `queryReference` | 0..1 | Reference | A relative literal (`Library/[id]`) or canonical (`[url]` or `[url]\|[version]`) reference to a stored Library. Resolves against the server's Library store. | +| `_format` | 0..1 | code | Output format. Accepts `ndjson` (default), `csv`, `json`, `parquet`, or `fhir` (or the corresponding media type). An unsupported explicit value returns `400`. | +| `header` | 0..1 | boolean | Include the header row in CSV output. Defaults to `true`. | +| `_limit` | 0..1 | integer | Maximum number of rows to return. Always clamped to the server-configured `maxRows` ceiling. | +| `parameters` | 0..1 | Parameters | Runtime parameter bindings. Each entry's name must match a `Library.parameter` declaration, and its `value[x]` must match the declared FHIR type. | Exactly one of `queryResource` and `queryReference` must be supplied to the system and type-level forms. The instance-level form ignores both and uses the Library identified in the URL. +The `source` parameter (an external data source) is not supported by this +server; supplying it returns `400 Bad Request` with an OperationOutcome at every +level, whether supplied in a POST `Parameters` body or as a GET query parameter. + +## Status codes + +| Status | Condition | +| --------------------------- | -------------------------------------------------------------------------------------------------------- | +| `200 OK` | A supported format (explicit or defaulted) and successful execution. | +| `400 Bad Request` | The unsupported `source` parameter, an unsupported explicit `_format`, or a malformed request/parameter. | +| `422 Unprocessable Entity` | `_format=fhir` where a result column has a type that cannot be represented as a FHIR value. | +| `500 Internal Server Error` | A genuine SQL execution or infrastructure fault. | + +When no `_format` is supplied, the format is negotiated from the `Accept` header, +falling back to NDJSON when nothing matches. + +The operation declares the SQL on FHIR spec canonical +`http://sql-on-fhir.org/OperationDefinition/$sqlquery-run` in the server +[CapabilityStatement](https://hl7.org/fhir/R4/capabilitystatement.html); the +server does not host a Pathling-authored OperationDefinition for it. + ## The Library resource The `$sqlquery-run` operation expects a Library that conforms to the SQLQuery @@ -201,6 +222,11 @@ When `_format` is `fhir`, the response is a FHIR Parameters resource Each column appears as a part with a typed `value[x]`, mapped from the SQL result schema. +If a result column has a type that cannot be represented as a FHIR value (for +example an array or struct), the server responds `422 Unprocessable Entity` with +an OperationOutcome identifying the column. The same query in a flat format +(`ndjson`, `csv`, `json`, or `parquet`) succeeds. + ## SQL constraints User SQL is validated before execution. The following are rejected: diff --git a/site/docs/server/operations/view-export.md b/site/docs/server/operations/view-export.md index e984e0c08e..df641949c3 100644 --- a/site/docs/server/operations/view-export.md +++ b/site/docs/server/operations/view-export.md @@ -22,16 +22,27 @@ POST [base]/$viewdefinition-export ## Parameters -| Name | Cardinality | Type | Description | -|---------------------|-------------|----------|--------------------------------------------------------------------------| -| `view.name` | 0..* | string | Optional names for exported views. Used in output filenames. | -| `view.viewResource` | 1..* | Resource | ViewDefinition resources to execute. | -| `clientTrackingId` | 0..1 | string | Client-provided tracking identifier for correlation. | -| `_format` | 0..1 | string | Output format: `ndjson` (default), `csv`, or `parquet`. | -| `header` | 0..1 | boolean | Include header row in CSV output. Defaults to `true`. | -| `patient` | 0..* | id | Filter to resources for the specified patient(s). | -| `group` | 0..* | id | Filter to resources for patients in the specified Group(s). | -| `_since` | 0..1 | instant | Only include resources where `meta.lastUpdated` is at or after this time.| +| Name | Cardinality | Type | Description | +| -------------------- | ----------- | --------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `view` | 1..\* | (parts) | A repeating parameter; each repetition identifies one ViewDefinition to export. | +| `view.name` | 0..1 | string | Optional name for the exported view. Used in output filenames. | +| `view.viewResource` | 0..1 | Resource | An inline ViewDefinition resource. Mutually exclusive with `view.viewReference`. | +| `view.viewReference` | 0..1 | Reference | A reference to a stored ViewDefinition. Mutually exclusive with `view.viewResource`. | +| `clientTrackingId` | 0..1 | string | Client-provided tracking identifier; echoed in the completion manifest. | +| `_format` | 0..1 | code | Output format: `ndjson` (default), `csv`, or `parquet`, or the corresponding media type. An unsupported value returns `400`. | +| `header` | 0..1 | boolean | Include header row in CSV output. Defaults to `true`. | +| `patient` | 0..\* | id | Filter to resources for the specified patient(s). | +| `group` | 0..\* | id | Filter to resources for patients in the specified Group(s). | +| `_since` | 0..1 | instant | Only include resources where `meta.lastUpdated` is at or after this time. | + +Each `view` part must supply exactly one of `view.viewResource` or +`view.viewReference`; supplying both, or neither, returns `400 Bad Request`. A +`view.viewReference` that does not resolve to a stored ViewDefinition returns +`404 Not Found`. These rejections are returned synchronously at kick-off. + +The `source` parameter (an external data source) is not supported by this +server; supplying it returns `400 Bad Request` with an OperationOutcome, +synchronously at kick-off. ## Asynchronous processing @@ -101,9 +112,21 @@ Prefer: respond-async ### Kick-off response +The `202 Accepted` response carries a `Parameters` acknowledgement with the +export status and identifier, and a `Content-Location` header pointing at the +status URL: + ```http HTTP/1.1 202 Accepted -Content-Location: [base]/$exportstatus/[job-id] +Content-Location: [base]/$job?id=[job-id] + +{ + "resourceType": "Parameters", + "parameter": [ + {"name": "status", "valueCode": "accepted"}, + {"name": "exportId", "valueString": "[job-id]"} + ] +} ``` ### Polling @@ -116,40 +139,61 @@ the export manifest. ## Response manifest -When the export completes, the response contains a JSON manifest: +When the export completes, the response is a FHIR `Parameters` resource +following the SQL on FHIR shape: ```json { - "transactionTime": "2025-01-15T10:30:00.000Z", - "request": "https://pathling.example.com/fhir/$viewdefinition-export", - "requiresAccessToken": true, - "output": [ + "resourceType": "Parameters", + "parameter": [ + { "name": "exportId", "valueString": "abc123" }, + { "name": "status", "valueCode": "completed" }, + { "name": "clientTrackingId", "valueString": "my-tracking-id" }, + { "name": "_format", "valueCode": "ndjson" }, + { + "name": "exportStartTime", + "valueInstant": "2026-06-21T01:00:00.000Z" + }, + { "name": "exportEndTime", "valueInstant": "2026-06-21T01:00:12.000Z" }, + { "name": "exportDuration", "valueInteger": 12 }, { - "name": "patient_demographics", - "url": "https://pathling.example.com/fhir/$result?job=abc123&file=patient_demographics.ndjson" + "name": "output", + "part": [ + { "name": "name", "valueString": "patient_demographics" }, + { + "name": "location", + "valueUri": "https://pathling.example.com/fhir/$result?job=abc123&file=patient_demographics.ndjson" + } + ] } - ], - "error": [] + ] } ``` ### Manifest fields -| Field | Description | -|-----------------------|-----------------------------------------------------------------| -| `transactionTime` | The time the export was initiated | -| `request` | The original kick-off request URL | -| `requiresAccessToken` | Whether authentication is required to download result files | -| `output` | Array of exported files with view name and download URL | -| `error` | Array of OperationOutcome files for any errors | +| Field | Description | +| ------------------ | ---------------------------------------------------------------------------------------- | +| `exportId` | The server-assigned export job identifier. | +| `status` | The export status; `completed` on the success path. | +| `clientTrackingId` | The tracking identifier supplied at kick-off. Present only when one was supplied. | +| `_format` | The effective output format. | +| `exportStartTime` | The time the export was initiated (kick-off). | +| `exportEndTime` | The time the export completed. | +| `exportDuration` | The whole number of seconds between the start and end times. | +| `output` | One per exported view, each with a `name` part and one or more `location` download URLs. | + +A view that produced several files (for example through Spark partitioning) is +represented as a single `output` with one `name` and a repeating `location` +part per file. ## Output formats -| Format | `_format` value | Content type | Description | -|-----------|-----------------|--------------------------------|---------------------------------------------| -| NDJSON | `ndjson` | `application/x-ndjson` | Newline-delimited JSON. Default format. | -| CSV | `csv` | `text/csv` | Comma-separated values. Use `header=false` to exclude headers. | -| Parquet | `parquet` | `application/vnd.apache.parquet` | Apache Parquet columnar format. Efficient for large datasets. | +| Format | `_format` value | Content type | Description | +| ------- | --------------- | -------------------------------- | -------------------------------------------------------------- | +| NDJSON | `ndjson` | `application/x-ndjson` | Newline-delimited JSON. Default format. | +| CSV | `csv` | `text/csv` | Comma-separated values. Use `header=false` to exclude headers. | +| Parquet | `parquet` | `application/vnd.apache.parquet` | Apache Parquet columnar format. Efficient for large datasets. | ## Multiple views @@ -244,13 +288,13 @@ Use `_since` to only include resources updated after a specific time: ## Comparison with run view -| Aspect | Export view | Run view | -|--------------------|--------------------------------------|---------------------------------------| -| Processing | Asynchronous with polling | Synchronous | -| Output | Files (download via manifest URLs) | Streamed response | -| Multiple views | Yes | No | -| Parquet format | Yes | No | -| Use case | Large datasets, batch processing | Small queries, interactive use | +| Aspect | Export view | Run view | +| -------------- | ---------------------------------- | ------------------------------ | +| Processing | Asynchronous with polling | Synchronous | +| Output | Files (download via manifest URLs) | Streamed response | +| Multiple views | Yes | No | +| Parquet format | Yes | No | +| Use case | Large datasets, batch processing | Small queries, interactive use | ## Python example @@ -388,24 +432,36 @@ def poll_status(status_url, timeout=3600): def download_files(manifest, output_dir): - """Download all files from the export manifest.""" + """Download all files from the export manifest Parameters resource.""" os.makedirs(output_dir, exist_ok=True) - for item in manifest.get("output", []): - name = item["name"] - url = item["url"] - # Extract extension from URL or default to ndjson. - ext = url.split(".")[-1] if "." in url.split("=")[-1] else "ndjson" - filename = f"{name}.{ext}" - filepath = os.path.join(output_dir, filename) - - print(f"Downloading {name}...") - response = requests.get(url) - response.raise_for_status() + # The manifest is a FHIR Parameters resource; each output parameter has a + # name part and one or more location parts (one per file). + for param in manifest.get("parameter", []): + if param.get("name") != "output": + continue + parts = param.get("part", []) + name = next( + (p.get("valueString") for p in parts if p.get("name") == "name"), + "output", + ) + locations = [ + p.get("valueUri") for p in parts if p.get("name") == "location" + ] + for index, url in enumerate(locations): + # Extract extension from the URL or default to ndjson. + ext = url.split(".")[-1] if "." in url.split("=")[-1] else "ndjson" + suffix = f"-{index}" if len(locations) > 1 else "" + filename = f"{name}{suffix}.{ext}" + filepath = os.path.join(output_dir, filename) + + print(f"Downloading {name}...") + response = requests.get(url) + response.raise_for_status() - with open(filepath, "wb") as f: - f.write(response.content) - print(f" Saved to {filepath}") + with open(filepath, "wb") as f: + f.write(response.content) + print(f" Saved to {filepath}") def main(): diff --git a/site/docs/server/operations/view-run.md b/site/docs/server/operations/view-run.md index 61e0a8f79a..3a1f47cb1a 100644 --- a/site/docs/server/operations/view-run.md +++ b/site/docs/server/operations/view-run.md @@ -13,22 +13,46 @@ follows the ## Endpoint +The operation is available at the system, type, and instance levels: + ``` POST [base]/$viewdefinition-run +POST [base]/ViewDefinition/$run +GET|POST [base]/ViewDefinition/[id]/$run ``` +At the system and type levels, supply exactly one of `viewResource` (inline) or +`viewReference` (a reference to a stored ViewDefinition). At the instance level, +the view is taken from the path. + ## Parameters -| Name | Cardinality | Type | Description | -| -------------- | ----------- | -------- | --------------------------------------------------------------------------------------------------------- | -| `viewResource` | 1..1 | Resource | The ViewDefinition resource. | -| `_format` | 0..1 | string | Output format. Accepts `application/x-ndjson` (default) or `text/csv`. | -| `header` | 0..1 | boolean | Include header row in CSV output. Defaults to `true`. | -| `_limit` | 0..1 | integer | Maximum number of rows to return. | -| `patient` | 0..\* | id | Filter to resources for the specified patient(s). | -| `group` | 0..\* | id | Filter to resources for patients in the specified Group(s). | -| `_since` | 0..1 | instant | Only include resources where `meta.lastUpdated` is at or after this time. | -| `resource` | 0..\* | string | Inline FHIR resources as JSON strings. When provided, these are used instead of the server's stored data. | +| Name | Cardinality | Type | Description | +| --------------- | ----------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `viewResource` | 0..1 | Resource | An inline ViewDefinition resource. Mutually exclusive with `viewReference`. | +| `viewReference` | 0..1 | Reference | A reference to a stored ViewDefinition (e.g. `ViewDefinition/patient-demographics`). Mutually exclusive with `viewResource`. | +| `_format` | 0..1 | code | Output format: `json`, `ndjson` (default), or `csv`, or the corresponding media type. An unsupported value returns `400`. | +| `header` | 0..1 | boolean | Include header row in CSV output. Defaults to `true`. | +| `_limit` | 0..1 | integer | Maximum number of rows to return. | +| `patient` | 0..1 | Reference | Filter to resources for the specified patient. More than one value returns `400`. | +| `group` | 0..\* | Reference | Filter to resources for patients in the specified Group(s). | +| `_since` | 0..1 | instant | Only include resources where `meta.lastUpdated` is at or after this time. | +| `resource` | 0..\* | Resource | Inline FHIR resources. A `Bundle` value is expanded to its entry resources. When provided, these are used instead of the server's stored data. | + +The `source` parameter (an external data source) is not supported by this +server; supplying it returns `400 Bad Request` with an OperationOutcome. + +## Status codes + +| Status | Condition | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `200 OK` | Success. The body is the rows in the negotiated format. | +| `400 Bad Request` | Malformed request or invalid parameter: an unsupported `_format`, the unsupported `source` parameter, both `viewResource` and `viewReference`, neither supplied (system/type), or more than one `patient`. | +| `404 Not Found` | A `viewReference` (or instance `[id]`) that does not resolve to a stored ViewDefinition. | +| `422 Unprocessable Entity` | A well-formed request carrying a semantically invalid ViewDefinition. | + +When no `_format` is supplied, the format is negotiated from the `Accept` header, +falling back to NDJSON when nothing matches. ## Request format @@ -82,6 +106,32 @@ Accept: application/x-ndjson } ``` +## Running a stored ViewDefinition by reference + +Instead of sending the ViewDefinition inline, you can run one that is already +stored on the server by supplying a `viewReference`: + +```http +POST [base]/ViewDefinition/$run HTTP/1.1 +Content-Type: application/fhir+json +Accept: application/x-ndjson + +{ + "resourceType": "Parameters", + "parameter": [ + { + "name": "viewReference", + "valueReference": { + "reference": "ViewDefinition/patient-demographics" + } + } + ] +} +``` + +The reference must be a literal relative reference of the form +`ViewDefinition/[id]`. A reference that does not resolve returns `404 Not Found`. + ## Response formats The response uses HTTP chunked transfer encoding, allowing clients to process @@ -206,8 +256,8 @@ testing ViewDefinitions or processing resources without importing them. ### Patient filter -Use the `patient` parameter to restrict results to resources associated with -specific patients: +Use the `patient` parameter (a single `Reference`) to restrict results to +resources associated with that patient: ```json { @@ -222,11 +272,9 @@ specific patients: }, { "name": "patient", - "valueId": "patient-123" - }, - { - "name": "patient", - "valueId": "patient-456" + "valueReference": { + "reference": "Patient/patient-123" + } } ] } @@ -235,7 +283,7 @@ specific patients: ### Group filter Use the `group` parameter to restrict results to resources for patients who are -members of the specified Group: +members of the specified Group(s). It accepts one or more `Reference` values: ```json { @@ -250,7 +298,9 @@ members of the specified Group: }, { "name": "group", - "valueId": "cohort-study-group" + "valueReference": { + "reference": "Group/cohort-study-group" + } } ] } From e27927ad3a65658bead0c22d208e41508d19ae85 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 19:50:19 +1000 Subject: [PATCH 007/162] test: Update view-export e2e mocks to the spec-shaped manifest Align the admin UI export end-to-end mocks with the new completion manifest: output download files are carried in location parts, and the non-spec transactionTime and requiresAccessToken fields are removed, so the card-listing tests exercise the location-reading parser. --- ui/e2e/sqlOnFhir.spec.ts | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/ui/e2e/sqlOnFhir.spec.ts b/ui/e2e/sqlOnFhir.spec.ts index 065a332af2..c606696b8e 100644 --- a/ui/e2e/sqlOnFhir.spec.ts +++ b/ui/e2e/sqlOnFhir.spec.ts @@ -540,17 +540,15 @@ test.describe("SQL on FHIR page", () => { body: JSON.stringify({ resourceType: "Parameters", parameter: [ - { - name: "transactionTime", - valueInstant: "2025-01-01T00:00:00Z", - }, - { name: "requiresAccessToken", valueBoolean: false }, + { name: "exportId", valueString: "test-job" }, + { name: "status", valueCode: "completed" }, + { name: "_format", valueCode: "ndjson" }, { name: "output", part: [ { name: "name", valueString: "patient_demographics" }, { - name: "url", + name: "location", valueUri: "http://localhost:3000/fhir/$result?job=test-job-123&file=patient_demographics.ndjson", }, @@ -620,18 +618,16 @@ test.describe("SQL on FHIR page", () => { body: JSON.stringify({ resourceType: "Parameters", parameter: [ - { - name: "transactionTime", - valueInstant: "2025-01-01T00:00:00Z", - }, - { name: "requiresAccessToken", valueBoolean: false }, + { name: "exportId", valueString: "test-job" }, + { name: "status", valueCode: "completed" }, + { name: "_format", valueCode: "ndjson" }, // Files intentionally in non-alphabetical order. { name: "output", part: [ { name: "name", valueString: "zebra_view" }, { - name: "url", + name: "location", valueUri: "http://localhost:3000/fhir/$result?job=test-job-456&file=zebra_view.ndjson", }, @@ -642,7 +638,7 @@ test.describe("SQL on FHIR page", () => { part: [ { name: "name", valueString: "alpha_view" }, { - name: "url", + name: "location", valueUri: "http://localhost:3000/fhir/$result?job=test-job-456&file=alpha_view.ndjson", }, @@ -653,7 +649,7 @@ test.describe("SQL on FHIR page", () => { part: [ { name: "name", valueString: "middle_view" }, { - name: "url", + name: "location", valueUri: "http://localhost:3000/fhir/$result?job=test-job-456&file=middle_view.ndjson", }, @@ -846,17 +842,15 @@ test.describe("SQL on FHIR page", () => { body: JSON.stringify({ resourceType: "Parameters", parameter: [ - { - name: "transactionTime", - valueInstant: "2025-01-01T00:00:00Z", - }, - { name: "requiresAccessToken", valueBoolean: false }, + { name: "exportId", valueString: "test-job" }, + { name: "status", valueCode: "completed" }, + { name: "_format", valueCode: "ndjson" }, { name: "output", part: [ { name: "name", valueString: "patient_demographics" }, { - name: "url", + name: "location", valueUri: "http://localhost:3000/fhir/$result?file=patient_demographics.ndjson", }, From 095ca72ecff8fe7923a52d3851c3bc11bc28203d Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 21 Jun 2026 20:01:21 +1000 Subject: [PATCH 008/162] fix: Accept a supported _format media type carrying parameters Strip media-type parameters (e.g. text/csv;charset=utf-8) before matching an explicit _format value across the run, export, and sqlquery-run strict parsers, so a supported media type with parameters is treated as that format rather than rejected with 400, matching the spec's content-negotiation edge case. --- .../operations/sqlquery/SqlQueryOutputFormat.java | 10 +++++++--- .../pathling/operations/view/ViewExportFormat.java | 6 ++++-- .../pathling/operations/view/ViewOutputFormat.java | 10 +++++++--- .../operations/sqlquery/SqlQueryOutputFormatTest.java | 7 +++++++ .../pathling/operations/view/ViewExportFormatTest.java | 7 +++++++ .../pathling/operations/view/ViewOutputFormatTest.java | 7 +++++++ 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java index c5f91cca4b..9cefb36a62 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormat.java @@ -101,12 +101,16 @@ public static SqlQueryOutputFormat fromStringStrict(@Nullable final String forma .formatted(format))); } - /** Matches a format string against the supported codes and content types. */ + /** + * Matches a format string against the supported codes and content types. Any media-type + * parameters (e.g. {@code text/csv;charset=utf-8}) are stripped before matching, so a supported + * media type carrying parameters is treated as that format. + */ @Nonnull private static Optional matchFormat(@Nonnull final String format) { - final String normalised = format.toLowerCase().trim(); + final String base = format.split(";", 2)[0].trim().toLowerCase(); return Arrays.stream(values()) - .filter(f -> f.code.equals(normalised) || f.contentType.equals(normalised)) + .filter(f -> f.code.equals(base) || f.contentType.equals(base)) .findFirst(); } diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java index d2a50fa73f..82d88cd55d 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewExportFormat.java @@ -72,9 +72,11 @@ public static ViewExportFormat fromString(@Nullable final String format) { if (format == null || format.isBlank()) { return NDJSON; } - final String normalised = format.toLowerCase().trim(); + // Strip any media-type parameters (e.g. "text/csv;charset=utf-8" -> "text/csv") so a supported + // media type carrying parameters is treated as that format. + final String base = format.split(";", 2)[0].trim().toLowerCase(); return Arrays.stream(values()) - .filter(f -> f.code.equals(normalised) || f.contentType.equals(normalised)) + .filter(f -> f.code.equals(base) || f.contentType.equals(base)) .findFirst() .orElseThrow( () -> diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java index ab633d96bc..5521a8acca 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewOutputFormat.java @@ -89,12 +89,16 @@ public static ViewOutputFormat fromStringStrict(@Nullable final String format) { .formatted(format))); } - /** Matches a format string against the supported codes and content types. */ + /** + * Matches a format string against the supported codes and content types. Any media-type + * parameters (e.g. {@code text/csv;charset=utf-8}) are stripped before matching, so a supported + * media type carrying parameters is treated as that format. + */ @Nonnull private static Optional matchFormat(@Nonnull final String format) { - final String normalised = format.toLowerCase().trim(); + final String base = format.split(";", 2)[0].trim().toLowerCase(); return Arrays.stream(values()) - .filter(f -> f.code.equals(normalised) || f.contentType.equals(normalised)) + .filter(f -> f.code.equals(base) || f.contentType.equals(base)) .findFirst(); } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java index 5faa72dd2e..f9ee1d1c43 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryOutputFormatTest.java @@ -163,6 +163,13 @@ void fromStringStrictDefaultsToNdjsonForBlank(final String input) { assertThat(SqlQueryOutputFormat.fromStringStrict(input)).isEqualTo(SqlQueryOutputFormat.NDJSON); } + @Test + void fromStringStrictAcceptsMediaTypeWithParameters() { + // A supported media type carrying parameters is treated as that format, not rejected. + assertThat(SqlQueryOutputFormat.fromStringStrict("text/csv;charset=utf-8")) + .isEqualTo(SqlQueryOutputFormat.CSV); + } + @Test void fromStringStrictRejectsUnknownNamingValue() { // An explicit unsupported format is rejected with the unsupported value named. diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java index e41cb221a0..9cb56c5ff7 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewExportFormatTest.java @@ -66,6 +66,13 @@ void parsesParquetContentType() { .isEqualTo(ViewExportFormat.PARQUET); } + @Test + void acceptsMediaTypeWithParameters() { + // A supported media type carrying parameters is treated as that format, not rejected. + assertThat(ViewExportFormat.fromString("text/csv;charset=utf-8")) + .isEqualTo(ViewExportFormat.CSV); + } + @Test void rejectsUnknownNamingValue() { // An explicit unsupported format is rejected with the unsupported value named. diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java index b295a1ee43..167cd928ab 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewOutputFormatTest.java @@ -105,6 +105,13 @@ void fromStringStrictDefaultsToNdjsonForBlank() { assertThat(ViewOutputFormat.fromStringStrict(" ")).isEqualTo(ViewOutputFormat.NDJSON); } + @Test + void fromStringStrictAcceptsMediaTypeWithParameters() { + // A supported media type carrying parameters is treated as that format, not rejected. + assertThat(ViewOutputFormat.fromStringStrict("text/csv;charset=utf-8")) + .isEqualTo(ViewOutputFormat.CSV); + } + @Test void fromStringStrictRejectsUnknownNamingValue() { // An explicit unsupported format is rejected with the unsupported value named. From 0b697abd37e31bc122a5db7bd7d0147ae38ac9fd Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 07:26:38 +1000 Subject: [PATCH 009/162] fix: Allow query parameter placeholders in SQL validation The strict SQL validator's expression allow-list omitted the named and positional parameter placeholder expressions, so any parameterised query (using ":name") failed static validation. This blocked runtime parameter binding for $sqlquery-run even though the binding machinery was already present. Add NamedParameter and PosParameter to the allow-list. They are leaf, unevaluable value placeholders bound at execution time, so they carry no code-execution risk. --- .../au/csiro/pathling/operations/sqlquery/SqlValidator.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java index 99785493aa..a6b008fceb 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java @@ -176,6 +176,11 @@ public class SqlValidator { "org.apache.spark.sql.catalyst.expressions.VariableReference", "org.apache.spark.sql.catalyst.expressions.AttributeReference", "org.apache.spark.sql.catalyst.expressions.BoundReference", + // Named and positional query parameters (e.g. ":name", "?"). These are leaf, unevaluable + // value placeholders bound at execution time via the parameterised SQL API, so they carry + // no code-execution risk. + "org.apache.spark.sql.catalyst.analysis.NamedParameter", + "org.apache.spark.sql.catalyst.analysis.PosParameter", // Arithmetic. "org.apache.spark.sql.catalyst.expressions.Add", "org.apache.spark.sql.catalyst.expressions.Subtract", From ce68015bbc149de17a73e5bcccc5df9c82464efa Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 07:26:54 +1000 Subject: [PATCH 010/162] feat: Add the $sqlquery-export operation Implement the SQL on FHIR $sqlquery-export operation, the asynchronous counterpart to $sqlquery-run, at the system, type, and instance levels. Clients can run one or more SQL queries against materialised ViewDefinition tables in the background and download each result as files, following the FHIR Asynchronous Request Pattern. Each query produces one output. Table sources can be supplied at request time, inline or by reference, in addition to being read from server storage. Supported formats are NDJSON (default), CSV, and Parquet, with patient, group, and _since filtering. A failed query fails the whole export. To avoid duplication, extract the shared asynchronous-export machinery into a new operations/export package (a generic completion-manifest builder, a file writer, and a filtered data-source builder) reused by both export operations, and refactor $sqlquery-run onto a shared SqlQueryPipeline that the export also uses. The Admin UI gains an export affordance on the SQL query result card, backed by a reusable export-job card shared with the ViewDefinition export. The operation is declared in the CapabilityStatement against the spec canonical and gated by a new sqlQueryExportEnabled configuration flag. Resolves #2633. --- .../java/au/csiro/pathling/FhirServer.java | 26 +- .../config/OperationConfiguration.java | 16 +- .../pathling/fhir/ConformanceProvider.java | 12 + .../export/ExportDataSourceBuilder.java | 109 +++++ .../operations/export/ExportFileWriter.java | 220 ++++++++++ .../operations/export/ExportManifest.java | 176 ++++++++ .../export/ExportManifestOutput.java | 33 ++ .../operations/sqlquery/PreparedSqlQuery.java | 39 ++ .../operations/sqlquery/QueryInput.java | 54 +++ .../sqlquery/SqlQueryExecutionHelper.java | 41 +- .../sqlquery/SqlQueryExportExecutor.java | 133 ++++++ .../sqlquery/SqlQueryExportProvider.java | 124 ++++++ .../sqlquery/SqlQueryExportRequest.java | 51 +++ .../sqlquery/SqlQueryExportRequestParser.java | 350 +++++++++++++++ .../sqlquery/SqlQueryExportSupport.java | 349 +++++++++++++++ .../SqlQueryInstanceExportProvider.java | 196 +++++++++ .../operations/sqlquery/SqlQueryPipeline.java | 148 +++++++ .../operations/sqlquery/ViewResolver.java | 32 +- .../view/ViewDefinitionExportExecutor.java | 237 ++--------- .../view/ViewDefinitionExportResponse.java | 103 +---- server/src/main/resources/application.yml | 1 + .../fhir/ConformanceProviderTest.java | 21 + .../operations/export/ExportManifestTest.java | 222 ++++++++++ .../sqlquery/AbstractSqlQueryExportIT.java | 363 ++++++++++++++++ .../sqlquery/RequestViewResolutionTest.java | 135 ++++++ .../sqlquery/SqlQueryExportConformanceIT.java | 79 ++++ .../sqlquery/SqlQueryExportDisabledIT.java | 79 ++++ .../sqlquery/SqlQueryExportExecutorTest.java | 182 ++++++++ .../sqlquery/SqlQueryExportFormatIT.java | 260 +++++++++++ .../sqlquery/SqlQueryExportInstanceIT.java | 82 ++++ .../sqlquery/SqlQueryExportProviderIT.java | 402 ++++++++++++++++++ .../SqlQueryExportRequestParserTest.java | 184 ++++++++ .../SqlQueryExportTestConfiguration.java | 242 +++++++++++ .../sqlquery/SqlQueryPipelineTest.java | 129 ++++++ .../ViewDefinitionExportExecutorTest.java | 55 ++- site/docs/server/configuration.md | 3 + site/docs/server/operations/sql-export.md | 231 ++++++++++ ui/CONTRIBUTING.md | 7 + ui/e2e/sqlQueryExport.spec.ts | 194 +++++++++ ui/src/api/__tests__/sqlQuery.test.ts | 86 +++- ui/src/api/index.ts | 5 + ui/src/api/sqlQuery.ts | 113 +++++ ui/src/components/sqlOnFhir/ExportJobCard.tsx | 262 ++++++++++++ ui/src/components/sqlOnFhir/SqlQueryCard.tsx | 70 ++- .../sqlOnFhir/SqlQueryExportCardWrapper.tsx | 136 ++++++ .../components/sqlOnFhir/ViewExportCard.tsx | 202 +-------- .../sqlOnFhir/__tests__/SqlQueryCard.test.tsx | 22 +- .../__tests__/sqlQueryExportHelpers.test.ts | 141 ++++++ ui/src/hooks/index.ts | 10 + ui/src/hooks/sqlQueryExportHelpers.ts | 118 +++++ ui/src/hooks/useSqlQueryExport.ts | 99 +++++ ui/src/types/job.ts | 13 +- ui/src/types/sqlQuery.ts | 34 ++ 53 files changed, 6099 insertions(+), 532 deletions(-) create mode 100644 server/src/main/java/au/csiro/pathling/operations/export/ExportDataSourceBuilder.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/export/ExportFileWriter.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/export/ExportManifest.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/export/ExportManifestOutput.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/QueryInput.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutor.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequest.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/export/ExportManifestTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/AbstractSqlQueryExportIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportConformanceIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportDisabledIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportInstanceIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java create mode 100644 site/docs/server/operations/sql-export.md create mode 100644 ui/e2e/sqlQueryExport.spec.ts create mode 100644 ui/src/components/sqlOnFhir/ExportJobCard.tsx create mode 100644 ui/src/components/sqlOnFhir/SqlQueryExportCardWrapper.tsx create mode 100644 ui/src/hooks/__tests__/sqlQueryExportHelpers.test.ts create mode 100644 ui/src/hooks/sqlQueryExportHelpers.ts create mode 100644 ui/src/hooks/useSqlQueryExport.ts diff --git a/server/src/main/java/au/csiro/pathling/FhirServer.java b/server/src/main/java/au/csiro/pathling/FhirServer.java index 1579f4a368..07375cbd81 100644 --- a/server/src/main/java/au/csiro/pathling/FhirServer.java +++ b/server/src/main/java/au/csiro/pathling/FhirServer.java @@ -182,6 +182,14 @@ public class FhirServer extends RestfulServer { private final transient au.csiro.pathling.operations.sqlquery.SqlQueryInstanceRunProvider sqlQueryInstanceRunProvider; + @Nonnull + private final transient au.csiro.pathling.operations.sqlquery.SqlQueryExportProvider + sqlQueryExportProvider; + + @Nonnull + private final transient au.csiro.pathling.operations.sqlquery.SqlQueryInstanceExportProvider + sqlQueryInstanceExportProvider; + /** * Constructs a new FhirServer. * @@ -213,6 +221,9 @@ public class FhirServer extends RestfulServer { * @param viewDefinitionExportProvider the view definition export provider * @param sqlQueryRunProvider the SQL query run provider * @param sqlQueryInstanceRunProvider the SQL query instance run provider + * @param sqlQueryExportProvider the system-level SQL query export provider + * @param sqlQueryInstanceExportProvider the type-level and instance-level SQL query export + * provider */ @SuppressWarnings("java:S107") public FhirServer( @@ -245,7 +256,12 @@ public FhirServer( @Nonnull final au.csiro.pathling.operations.sqlquery.SqlQueryRunProvider sqlQueryRunProvider, @Nonnull final au.csiro.pathling.operations.sqlquery.SqlQueryInstanceRunProvider - sqlQueryInstanceRunProvider) { + sqlQueryInstanceRunProvider, + @Nonnull + final au.csiro.pathling.operations.sqlquery.SqlQueryExportProvider sqlQueryExportProvider, + @Nonnull + final au.csiro.pathling.operations.sqlquery.SqlQueryInstanceExportProvider + sqlQueryInstanceExportProvider) { // Pass the FhirContext to the RestfulServer superclass to ensure custom types like // ViewDefinitionResource are recognized when parsing request bodies. super(fhirContext); @@ -276,6 +292,8 @@ public FhirServer( this.viewDefinitionExportProvider = viewDefinitionExportProvider; this.sqlQueryRunProvider = sqlQueryRunProvider; this.sqlQueryInstanceRunProvider = sqlQueryInstanceRunProvider; + this.sqlQueryExportProvider = sqlQueryExportProvider; + this.sqlQueryInstanceExportProvider = sqlQueryInstanceExportProvider; } @Override @@ -400,6 +418,12 @@ protected void initialize() throws ServletException { registerProvider(sqlQueryInstanceRunProvider); } + // Register the SQL query export providers (system, type, and instance levels). + if (ops.isSqlQueryExportEnabled()) { + registerProvider(sqlQueryExportProvider); + registerProvider(sqlQueryInstanceExportProvider); + } + // CORS configuration. configureCors(); diff --git a/server/src/main/java/au/csiro/pathling/config/OperationConfiguration.java b/server/src/main/java/au/csiro/pathling/config/OperationConfiguration.java index ce82e00113..2799fa398b 100644 --- a/server/src/main/java/au/csiro/pathling/config/OperationConfiguration.java +++ b/server/src/main/java/au/csiro/pathling/config/OperationConfiguration.java @@ -73,15 +73,25 @@ public class OperationConfiguration { /** Enables $sqlquery-run operation. */ private boolean sqlQueryRunEnabled = true; + /** Enables $sqlquery-export operation. */ + private boolean sqlQueryExportEnabled = true; + /** Enables $bulk-submit operation. */ private boolean bulkSubmitEnabled = true; /** - * Returns true if any export operation is enabled. + * Returns true if any operation that serves its results through the {@code $result} endpoint is + * enabled. This covers the Bulk Data exports as well as the SQL on FHIR asynchronous export + * operations ({@code $viewdefinition-export} and {@code $sqlquery-export}), all of which write + * downloadable files served by {@code $result}. * - * @return true if system, patient, or group export is enabled + * @return true if any export operation that relies on the {@code $result} endpoint is enabled */ public boolean isAnyExportEnabled() { - return exportEnabled || patientExportEnabled || groupExportEnabled; + return exportEnabled + || patientExportEnabled + || groupExportEnabled + || viewDefinitionExportEnabled + || sqlQueryExportEnabled; } } diff --git a/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java b/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java index a13d2a4284..8a396f8586 100644 --- a/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java +++ b/server/src/main/java/au/csiro/pathling/fhir/ConformanceProvider.java @@ -107,6 +107,7 @@ public class ConformanceProvider private static final String VIEWDEFINITION_RUN_OPERATION = "viewdefinition-run"; private static final String VIEWDEFINITION_EXPORT_OPERATION = "viewdefinition-export"; private static final String SQLQUERY_RUN_OPERATION = "sqlquery-run"; + private static final String SQLQUERY_EXPORT_OPERATION = "sqlquery-export"; /** * The spec canonical OperationDefinition URLs for the SQL on FHIR operations. The server declares @@ -122,6 +123,9 @@ public class ConformanceProvider private static final String SOF_SQLQUERY_RUN_CANONICAL = "http://sql-on-fhir.org/OperationDefinition/$sqlquery-run"; + private static final String SOF_SQLQUERY_EXPORT_CANONICAL = + "http://sql-on-fhir.org/OperationDefinition/$sqlquery-export"; + /** * Base system-level operations whose Pathling-authored OperationDefinition resources are served. * The SQL on FHIR run/export operations are intentionally excluded: they declare the spec @@ -533,6 +537,14 @@ private List buildOperations( addOperationIfEnabled( operations, SQLQUERY_RUN_OPERATION, ops.isSqlQueryRunEnabled(), SOF_SQLQUERY_RUN_CANONICAL); + // Add SQL query export operation, declaring the SQL on FHIR spec canonical. The referenced + // OperationDefinition declares the system, type, and instance scopes. + addOperationIfEnabled( + operations, + SQLQUERY_EXPORT_OPERATION, + ops.isSqlQueryExportEnabled(), + SOF_SQLQUERY_EXPORT_CANONICAL); + // Add bulk submit operations if configured and enabled. if (configuration.getBulkSubmit() != null && ops.isBulkSubmitEnabled()) { addOperationIfEnabled(operations, "bulk-submit", true); diff --git a/server/src/main/java/au/csiro/pathling/operations/export/ExportDataSourceBuilder.java b/server/src/main/java/au/csiro/pathling/operations/export/ExportDataSourceBuilder.java new file mode 100644 index 0000000000..97a2b9e3dd --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/export/ExportDataSourceBuilder.java @@ -0,0 +1,109 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.export; + +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Set; +import org.hl7.fhir.r4.model.InstantType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Builds the filtered data source for an asynchronous export: applies the {@code _since} + * (updated-since) filter and the patient-compartment filter derived from the {@code patient} and + * {@code group} parameters. Shared by both {@code $viewdefinition-export} and {@code + * $sqlquery-export} so the two operations scope exported rows identically. + * + * @author John Grimes + */ +@Component +public class ExportDataSourceBuilder { + + @Nonnull private final PatientCompartmentService patientCompartmentService; + + /** + * Constructs a new ExportDataSourceBuilder. + * + * @param patientCompartmentService the patient compartment service used for row-level filtering + */ + @Autowired + public ExportDataSourceBuilder( + @Nonnull final PatientCompartmentService patientCompartmentService) { + this.patientCompartmentService = patientCompartmentService; + } + + /** + * Applies the export filters to the base data source. + * + * @param base the unfiltered data source + * @param since the {@code _since} filter, or null for no time filter + * @param patientIds the patient ids (resolved from {@code patient} and {@code group}); empty for + * no compartment filter + * @return the filtered data source + */ + @Nonnull + public QueryableDataSource build( + @Nonnull final QueryableDataSource base, + @Nullable final InstantType since, + @Nonnull final Set patientIds) { + QueryableDataSource dataSource = base; + + // Apply the _since filter. + if (since != null) { + dataSource = + dataSource.map( + rowDataset -> + rowDataset.filter( + "meta.lastUpdated IS NULL OR meta.lastUpdated >= '" + + since.getValueAsString() + + "'")); + } + + // Apply the patient compartment filter if patient ids were specified. + if (!patientIds.isEmpty()) { + dataSource = applyPatientCompartmentFilter(dataSource, patientIds); + } + + return dataSource; + } + + /** + * Applies the patient compartment filter to the data source. Uses the FHIRPath-aware compartment + * filter so that non-Patient resource types whose compartment membership is defined by a FHIRPath + * expression (for example {@code Observation.subject.where(resolve() is Patient)}) are filtered + * correctly, not just resources with a flat reference column. + */ + @Nonnull + private QueryableDataSource applyPatientCompartmentFilter( + @Nonnull final QueryableDataSource base, @Nonnull final Set patientIds) { + + // Filter out resource types that are not in the Patient compartment. + final QueryableDataSource filtered = + base.filterByResourceType(patientCompartmentService::isInPatientCompartment); + + // Apply row-level filtering based on patient compartment membership, evaluating any FHIRPath + // compartment paths against the unfiltered source. + return filtered.map( + (resourceType, rowDataset) -> + patientCompartmentService.filterByPatientCompartment( + resourceType, patientIds, rowDataset, base)); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/export/ExportFileWriter.java b/server/src/main/java/au/csiro/pathling/operations/export/ExportFileWriter.java new file mode 100644 index 0000000000..f49c667f39 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/export/ExportFileWriter.java @@ -0,0 +1,220 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.export; + +import static au.csiro.pathling.library.io.FileSystemPersistence.safelyJoinPaths; + +import au.csiro.pathling.library.io.FileSystemPersistence; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import jakarta.annotation.Nonnull; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.sql.AnalysisException; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * Writes asynchronous-export result datasets to files under the per-job directory in the warehouse, + * and serves the per-job directory and unique-naming helpers. Shared by both {@code + * $viewdefinition-export} and {@code $sqlquery-export} so that the two operations write their + * outputs identically (the same directory layout, partition renaming, and CSV unsupported-type + * handling). + * + * @author John Grimes + */ +@Slf4j +@Component +public class ExportFileWriter { + + @Nonnull private final SparkSession sparkSession; + + @Nonnull private final String databasePath; + + /** + * Constructs a new ExportFileWriter. + * + * @param sparkSession the Spark session used to write the output files + * @param databasePath the warehouse database path under which the {@code jobs} directory lives + */ + @Autowired + public ExportFileWriter( + @Nonnull final SparkSession sparkSession, + @Nonnull @Value("${pathling.storage.warehouseUrl}/${pathling.storage.databaseName}") + final String databasePath) { + this.sparkSession = sparkSession; + this.databasePath = databasePath; + } + + /** + * Creates the per-job directory under the warehouse for storing output files. + * + * @param jobId the job id + * @return the job directory path + */ + @Nonnull + public Path createJobDirectory(@Nonnull final String jobId) { + final URI warehouseUri = URI.create(databasePath); + final Path warehousePath = new Path(warehouseUri); + final Path jobDirPath = new Path(new Path(warehousePath, "jobs"), jobId); + final Configuration configuration = sparkSession.sparkContext().hadoopConfiguration(); + + try { + final FileSystem fs = FileSystem.get(configuration); + if (!fs.exists(jobDirPath)) { + final boolean created = fs.mkdirs(jobDirPath); + if (!created) { + throw new InternalErrorException( + "Failed to create subdirectory at %s for job %s.".formatted(databasePath, jobId)); + } + log.debug("Created dir {}", jobDirPath); + } + } catch (final IOException e) { + throw new InternalErrorException( + "Failed to create subdirectory at %s for job %s.".formatted(databasePath, jobId)); + } + + return jobDirPath; + } + + /** + * Deletes the per-job directory and all its contents, used to clean up partial outputs when an + * export fails. Failures to delete are logged and swallowed, since cleanup is best-effort. + * + * @param jobId the job id whose directory should be removed + */ + public void deleteJobDirectory(@Nonnull final String jobId) { + final URI warehouseUri = URI.create(databasePath); + final Path jobDirPath = new Path(new Path(new Path(warehouseUri), "jobs"), jobId); + try { + final FileSystem fs = FileSystem.get(sparkSession.sparkContext().hadoopConfiguration()); + if (fs.exists(jobDirPath)) { + fs.delete(jobDirPath, /* recursive= */ true); + log.debug("Deleted partial output dir {}", jobDirPath); + } + } catch (final IOException e) { + log.warn("Failed to delete partial output directory for job {}", jobId, e); + } + } + + /** + * Returns a name unique among the already-used names, appending a numeric suffix on collision. + * + * @param baseName the desired base name + * @param usedNames the names already used in this export (not modified) + * @return a unique name + */ + @Nonnull + public String uniqueName(@Nonnull final String baseName, @Nonnull final Set usedNames) { + if (!usedNames.contains(baseName)) { + return baseName; + } + int suffix = 1; + while (usedNames.contains(baseName + "_" + suffix)) { + suffix++; + } + return baseName + "_" + suffix; + } + + /** + * Writes the result as NDJSON files, returning the resulting file URLs. + * + * @param result the result dataset + * @param name the output name (used as the file/directory base name) + * @param jobDirPath the per-job directory + * @return the written file URLs, one per partition + */ + @Nonnull + public List writeNdjson( + @Nonnull final Dataset result, + @Nonnull final String name, + @Nonnull final Path jobDirPath) { + final String outputPath = safelyJoinPaths(jobDirPath.toString(), name + ".ndjson"); + result.write().mode(SaveMode.Overwrite).json(outputPath); + return new ArrayList<>( + FileSystemPersistence.renamePartitionedFiles(sparkSession, outputPath, outputPath, "json")); + } + + /** + * Writes the result as CSV files, returning the resulting file URLs. + * + * @param result the result dataset + * @param name the output name (used as the file/directory base name) + * @param includeHeader whether to include a CSV header row + * @param jobDirPath the per-job directory + * @return the written file URLs, one per partition + * @throws InvalidRequestException if the dataset contains data types unsupported by CSV + */ + @Nonnull + public List writeCsv( + @Nonnull final Dataset result, + @Nonnull final String name, + final boolean includeHeader, + @Nonnull final Path jobDirPath) { + final String outputPath = safelyJoinPaths(jobDirPath.toString(), name + ".csv"); + try { + result.write().mode(SaveMode.Overwrite).option("header", includeHeader).csv(outputPath); + } catch (final Exception e) { + // Spark throws AnalysisException when it encounters unsupported data types for a datasource. + // We convert this to an InvalidRequestException to return a 400 status code. + if (e instanceof final AnalysisException ae + && "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE".equals(ae.getErrorClass())) { + throw new InvalidRequestException( + "CSV export failed for output '%s': %s".formatted(name, e.getMessage())); + } + if (e instanceof final RuntimeException re) { + throw re; + } + throw new RuntimeException(e); + } + return new ArrayList<>( + FileSystemPersistence.renamePartitionedFiles(sparkSession, outputPath, outputPath, "csv")); + } + + /** + * Writes the result as Parquet files, returning the resulting file URLs. + * + * @param result the result dataset + * @param name the output name (used as the file/directory base name) + * @param jobDirPath the per-job directory + * @return the written file URLs, one per partition + */ + @Nonnull + public List writeParquet( + @Nonnull final Dataset result, + @Nonnull final String name, + @Nonnull final Path jobDirPath) { + final String outputPath = safelyJoinPaths(jobDirPath.toString(), name + ".parquet"); + result.write().mode(SaveMode.Overwrite).parquet(outputPath); + return new ArrayList<>( + FileSystemPersistence.renamePartitionedFiles( + sparkSession, outputPath, outputPath, "parquet")); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/export/ExportManifest.java b/server/src/main/java/au/csiro/pathling/operations/export/ExportManifest.java new file mode 100644 index 0000000000..65af533e59 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/export/ExportManifest.java @@ -0,0 +1,176 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.export; + +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.net.URISyntaxException; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.List; +import org.apache.http.client.utils.URIBuilder; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.UriType; + +/** + * Builds the SQL on FHIR asynchronous-export completion manifest as a FHIR {@code Parameters} + * resource. The manifest follows the shape shared by {@code $viewdefinition-export} and {@code + * $sqlquery-export}: a required {@code exportId} and {@code status}, the echoed {@code + * clientTrackingId} and {@code _format}, the export timing fields, and one {@code output} per + * exported unit with a {@code name} and one or more {@code location} download URLs. + * + *

    The {@code cancelUrl} and {@code estimatedTimeRemaining} parameters are deliberately omitted, + * consistent with the agreed scope of both export operations. + * + * @author John Grimes + */ +public class ExportManifest { + + @Nonnull private final String serverBaseUrl; + + @Nonnull private final String exportId; + + @Nullable private final String clientTrackingId; + + @Nonnull private final String format; + + @Nonnull private final Instant exportStartTime; + + @Nonnull private final Instant exportEndTime; + + @Nonnull private final List outputs; + + /** + * Creates a new ExportManifest. + * + * @param serverBaseUrl the FHIR server base URL (used for constructing the result download URLs) + * @param exportId the server-assigned export job identifier + * @param clientTrackingId the client-supplied tracking identifier, or null if none was supplied + * @param format the effective output format code (e.g. {@code ndjson}) + * @param exportStartTime the export job creation (kick-off) time + * @param exportEndTime the time the manifest is built on completion + * @param outputs the export outputs, one per exported unit, in order + */ + @SuppressWarnings("java:S107") + public ExportManifest( + @Nonnull final String serverBaseUrl, + @Nonnull final String exportId, + @Nullable final String clientTrackingId, + @Nonnull final String format, + @Nonnull final Instant exportStartTime, + @Nonnull final Instant exportEndTime, + @Nonnull final List outputs) { + this.serverBaseUrl = serverBaseUrl; + this.exportId = exportId; + this.clientTrackingId = clientTrackingId; + this.format = format; + this.exportStartTime = exportStartTime; + this.exportEndTime = exportEndTime; + this.outputs = outputs; + } + + /** + * Builds the completion manifest as a FHIR {@code Parameters} resource. + * + * @return the manifest Parameters + */ + @Nonnull + public Parameters toParameters() { + final Parameters parameters = new Parameters(); + + // Ensure the base URL ends with a slash for proper URL construction. + final String normalizedBaseUrl = + serverBaseUrl.endsWith("/") ? serverBaseUrl : serverBaseUrl + "/"; + + parameters.addParameter().setName("exportId").setValue(new StringType(exportId)); + parameters.addParameter().setName("status").setValue(new CodeType("completed")); + + // Echo the client tracking id only when one was supplied at kick-off. + if (clientTrackingId != null && !clientTrackingId.isBlank()) { + parameters + .addParameter() + .setName("clientTrackingId") + .setValue(new StringType(clientTrackingId)); + } + + parameters.addParameter().setName("_format").setValue(new CodeType(format)); + + parameters + .addParameter() + .setName("exportStartTime") + .setValue(new InstantType(Date.from(exportStartTime))); + parameters + .addParameter() + .setName("exportEndTime") + .setValue(new InstantType(Date.from(exportEndTime))); + + // Whole seconds between start and end, never negative. + final long durationSeconds = + Math.max(0L, Duration.between(exportStartTime, exportEndTime).toSeconds()); + parameters + .addParameter() + .setName("exportDuration") + .setValue(new IntegerType((int) durationSeconds)); + + // One output per exported unit, with a name and one or more location parts. + for (final ExportManifestOutput output : outputs) { + final ParametersParameterComponent outputParam = parameters.addParameter().setName("output"); + outputParam.addPart().setName("name").setValue(new StringType(output.name())); + for (final String fileUrl : output.fileUrls()) { + outputParam + .addPart() + .setName("location") + .setValue(new UriType(buildResultUrl(normalizedBaseUrl, fileUrl))); + } + } + + return parameters; + } + + /** + * Converts a local file URL to a remote {@code $result} download URL. + * + * @param baseUrl the normalised base server URL (ending in a slash) + * @param localUrl the local file URL containing the job id and filename + * @return the remote result URL + */ + @Nonnull + private static String buildResultUrl( + @Nonnull final String baseUrl, @Nonnull final String localUrl) { + try { + final String[] parts = localUrl.split("/jobs/")[1].split("/"); + final String jobUuid = parts[0]; + final String file = parts[1]; + + return new URIBuilder(baseUrl + "$result") + .addParameter("job", jobUuid) + .addParameter("file", file) + .build() + .toString(); + } catch (final URISyntaxException e) { + throw new InternalErrorException(e); + } + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/export/ExportManifestOutput.java b/server/src/main/java/au/csiro/pathling/operations/export/ExportManifestOutput.java new file mode 100644 index 0000000000..a3d7b85eb8 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/export/ExportManifestOutput.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.export; + +import jakarta.annotation.Nonnull; +import java.util.List; + +/** + * One finished export unit within a completion manifest: a friendly name and one or more + * downloadable file locations (local file URLs that the manifest builder maps to {@code $result} + * download URLs). Shared by both the {@code $viewdefinition-export} and {@code $sqlquery-export} + * operations, which produce the identical manifest shape (one output per exported unit). + * + * @param name the output name (one per exported unit) + * @param fileUrls the local file URLs for this output, in order; repeats once per partitioned file + * @author John Grimes + */ +public record ExportManifestOutput(@Nonnull String name, @Nonnull List fileUrls) {} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java new file mode 100644 index 0000000000..a89abf1e49 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.views.FhirView; +import jakarta.annotation.Nonnull; +import java.util.Map; +import lombok.Value; + +/** + * A SQL query that has been parsed and had its ViewDefinition table sources resolved, ready for + * static validation and execution by {@link SqlQueryPipeline}. Produced by {@link + * SqlQueryPipeline#prepare}; shared by the synchronous {@code $sqlquery-run} and the asynchronous + * {@code $sqlquery-export} operations. + */ +@Value +public class PreparedSqlQuery { + + /** The validated, normalised request: parsed query, output format, header flag, bindings. */ + @Nonnull SqlQueryRequest request; + + /** The resolved view table sources the SQL references, keyed by table label. */ + @Nonnull Map resolvedViews; +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/QueryInput.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/QueryInput.java new file mode 100644 index 0000000000..c162ddf727 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/QueryInput.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; + +/** + * One {@code query} repetition of a {@code $sqlquery-export} request, prepared (parsed, parameter + * bound, and view resolved) at kick-off and carried to background execution. Each query input + * produces exactly one export output. + * + * @param name the optional {@code query.name}, the highest-precedence output name + * @param libraryName the SQLQuery Library's {@code name} element, used as the output-name fallback + * @param preparedQuery the prepared query (parsed SQL, bound parameters, resolved views) + * @author John Grimes + */ +public record QueryInput( + @Nullable String name, @Nullable String libraryName, @Nonnull PreparedSqlQuery preparedQuery) { + + /** + * Derives the output name for this query: the {@code query.name} when supplied, otherwise the + * SQLQuery Library's {@code name} element, otherwise a generated name based on the index. Names + * are made unique across the export by the executor. + * + * @param index the index of this query in the request (used for the generated fallback) + * @return the effective output name before uniqueness is applied + */ + @Nonnull + public String getEffectiveName(final int index) { + if (name != null && !name.isBlank()) { + return name; + } + if (libraryName != null && !libraryName.isBlank()) { + return libraryName; + } + return "query_" + index; + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java index 50d1a1e4d6..b25ace4c17 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutionHelper.java @@ -18,7 +18,6 @@ package au.csiro.pathling.operations.sqlquery; import au.csiro.pathling.library.io.source.QueryableDataSource; -import au.csiro.pathling.views.FhirView; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; @@ -33,19 +32,15 @@ import org.springframework.stereotype.Component; /** - * Orchestrates the {@code $sqlquery-run} operation by chaining the parser, view resolver, executor - * and result streamer. + * Orchestrates the {@code $sqlquery-run} operation by selecting the query Library, running it + * through the shared {@link SqlQueryPipeline}, and streaming the single result. * * @author John Grimes */ @Component public class SqlQueryExecutionHelper { - @Nonnull private final SqlQueryRequestParser requestParser; - - @Nonnull private final ViewResolver viewResolver; - - @Nonnull private final SqlQueryExecutor executor; + @Nonnull private final SqlQueryPipeline pipeline; @Nonnull private final SqlQueryResultStreamer streamer; @@ -56,25 +51,18 @@ public class SqlQueryExecutionHelper { /** * Constructs a new SqlQueryExecutionHelper. * - * @param requestParser parses raw HTTP inputs into a validated request - * @param viewResolver resolves view references to parsed FhirViews with auth checks - * @param executor validates and runs the SQL against Spark + * @param pipeline the shared SQL query pipeline (parse, resolve, validate, execute) * @param streamer streams the result dataset in the requested format * @param deltaLake the queryable data source backing FhirView execution * @param libraryReferenceResolver resolves a queryReference to a stored Library */ - @SuppressWarnings("java:S107") @Autowired public SqlQueryExecutionHelper( - @Nonnull final SqlQueryRequestParser requestParser, - @Nonnull final ViewResolver viewResolver, - @Nonnull final SqlQueryExecutor executor, + @Nonnull final SqlQueryPipeline pipeline, @Nonnull final SqlQueryResultStreamer streamer, @Nonnull final QueryableDataSource deltaLake, @Nonnull final LibraryReferenceResolver libraryReferenceResolver) { - this.requestParser = requestParser; - this.viewResolver = viewResolver; - this.executor = executor; + this.pipeline = pipeline; this.streamer = streamer; this.deltaLake = deltaLake; this.libraryReferenceResolver = libraryReferenceResolver; @@ -128,20 +116,19 @@ public void executeSqlQuery( final IBaseResource library = selectLibrary(queryResource, queryReference); - final SqlQueryRequest request = - requestParser.parse(library, format, acceptHeader, includeHeader, limit, parameters); - - final Map resolvedViews = - viewResolver.resolve(request.getParsedQuery().getViewReferences()); + final PreparedSqlQuery prepared = + pipeline.prepare(library, format, acceptHeader, includeHeader, limit, parameters, Map.of()); - executor.execute( - request, - resolvedViews, + pipeline.execute( + prepared, deltaLake, requestId, result -> streamer.stream( - result, request.getOutputFormat(), request.isIncludeHeader(), response)); + result, + prepared.getRequest().getOutputFormat(), + prepared.getRequest().isIncludeHeader(), + response)); } /** diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutor.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutor.java new file mode 100644 index 0000000000..f107e56cc8 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutor.java @@ -0,0 +1,133 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.export.ExportDataSourceBuilder; +import au.csiro.pathling.operations.export.ExportFileWriter; +import au.csiro.pathling.operations.export.ExportManifestOutput; +import au.csiro.pathling.operations.view.ViewExportFormat; +import jakarta.annotation.Nonnull; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hadoop.fs.Path; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Runs each query of a {@code $sqlquery-export} request via the shared {@link SqlQueryPipeline} and + * writes the result of each to files, producing one output per query. Reuses the shared {@link + * ExportDataSourceBuilder} (filtering) and {@link ExportFileWriter} (job directory and file + * writing) that back {@code $viewdefinition-export}. + * + *

    Execution is all-or-nothing: if any query fails, the exception propagates and the whole export + * fails, so no completion manifest is produced. + * + * @author John Grimes + */ +@Component +public class SqlQueryExportExecutor { + + @Nonnull private final SqlQueryPipeline pipeline; + + @Nonnull private final QueryableDataSource deltaLake; + + @Nonnull private final ExportDataSourceBuilder dataSourceBuilder; + + @Nonnull private final ExportFileWriter fileWriter; + + /** + * Constructs a new SqlQueryExportExecutor. + * + * @param pipeline the shared SQL query pipeline (execution) + * @param deltaLake the queryable data source backing FhirView execution + * @param dataSourceBuilder the shared export data-source builder (applies filters) + * @param fileWriter the shared export file writer (job directory and file writing) + */ + @Autowired + public SqlQueryExportExecutor( + @Nonnull final SqlQueryPipeline pipeline, + @Nonnull final QueryableDataSource deltaLake, + @Nonnull final ExportDataSourceBuilder dataSourceBuilder, + @Nonnull final ExportFileWriter fileWriter) { + this.pipeline = pipeline; + this.deltaLake = deltaLake; + this.dataSourceBuilder = dataSourceBuilder; + this.fileWriter = fileWriter; + } + + /** + * Executes the export request and writes the results to files, one output per query. + * + * @param request the export request + * @param jobId the job id for this export + * @return the outputs, one per query, in order + */ + @Nonnull + public List execute( + @Nonnull final SqlQueryExportRequest request, @Nonnull final String jobId) { + + final Path jobDirPath = fileWriter.createJobDirectory(jobId); + final QueryableDataSource dataSource = + dataSourceBuilder.build(deltaLake, request.since(), request.patientIds()); + final List outputs = new ArrayList<>(); + final Set usedNames = new HashSet<>(); + + for (int i = 0; i < request.queries().size(); i++) { + final QueryInput query = request.queries().get(i); + final String outputName = fileWriter.uniqueName(query.getEffectiveName(i), usedNames); + usedNames.add(outputName); + + // Namespace the request-scoped temp views per query within the job. + final String requestId = jobId + "-" + i; + final AtomicReference> fileUrls = new AtomicReference<>(List.of()); + pipeline.execute( + query.preparedQuery(), + dataSource, + requestId, + result -> + fileUrls.set( + writeOutput( + result, outputName, request.format(), request.includeHeader(), jobDirPath))); + + outputs.add(new ExportManifestOutput(outputName, fileUrls.get())); + } + + return outputs; + } + + /** Writes the query result in the requested format via the shared file writer. */ + @Nonnull + private List writeOutput( + @Nonnull final Dataset result, + @Nonnull final String name, + @Nonnull final ViewExportFormat format, + final boolean includeHeader, + @Nonnull final Path jobDirPath) { + return switch (format) { + case NDJSON -> fileWriter.writeNdjson(result, name, jobDirPath); + case CSV -> fileWriter.writeCsv(result, name, includeHeader, jobDirPath); + case PARQUET -> fileWriter.writeParquet(result, name, jobDirPath); + }; + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java new file mode 100644 index 0000000000..a838f27dc7 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java @@ -0,0 +1,124 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.async.AsyncSupported; +import au.csiro.pathling.async.PreAsyncValidation; +import au.csiro.pathling.security.OperationAccess; +import ca.uhn.fhir.rest.annotation.Operation; +import ca.uhn.fhir.rest.annotation.OperationParam; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Reference; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Provider for the system-level {@code $sqlquery-export} operation from the SQL on FHIR + * specification: the asynchronous counterpart to {@code $sqlquery-run}. Runs one or more SQL + * queries against materialised ViewDefinition tables in the background and exports each result to + * downloadable files. The level-agnostic machinery is shared via {@link SqlQueryExportSupport}. + * + * @author John Grimes + * @see $sqlquery-export + * @see SqlQueryInstanceExportProvider for the type-level and instance-level operations + */ +@Slf4j +@Component +public class SqlQueryExportProvider implements PreAsyncValidation { + + @Nonnull private final SqlQueryExportRequestParser requestParser; + + @Nonnull private final SqlQueryExportSupport support; + + /** + * Constructs a new SqlQueryExportProvider. + * + * @param requestParser parses and validates the kick-off request + * @param support the shared export machinery (job resolution, execution, manifest, cache key) + */ + @Autowired + public SqlQueryExportProvider( + @Nonnull final SqlQueryExportRequestParser requestParser, + @Nonnull final SqlQueryExportSupport support) { + this.requestParser = requestParser; + this.support = support; + } + + /** + * Handles the {@code $sqlquery-export} operation at the system level. + * + * @param clientTrackingId optional client-provided tracking identifier + * @param format the output format (ndjson, csv, parquet) + * @param includeHeader whether to include headers in CSV output + * @param patientIds patient ids to filter by + * @param groupIds group ids to filter by + * @param since filter resources modified after this timestamp + * @param source the unsupported external data source parameter, rejected when supplied + * @param requestDetails the request details + * @return the completion manifest, or null if cancelled + */ + @SuppressWarnings({"unused", "java:S107"}) + @Operation(name = "$sqlquery-export", idempotent = true) + @OperationAccess("sqlquery-export") + @AsyncSupported(redirectOnComplete = true) + @Nullable + public Parameters export( + @Nullable @OperationParam(name = "clientTrackingId") final String clientTrackingId, + @Nullable @OperationParam(name = "_format") final String format, + @Nullable @OperationParam(name = "header") final BooleanType includeHeader, + @Nullable @OperationParam(name = "patient") final List patientIds, + @Nullable @OperationParam(name = "group") final List groupIds, + @Nullable @OperationParam(name = "_since") final InstantType since, + @Nullable @OperationParam(name = "source") final String source, + @Nonnull final ServletRequestDetails requestDetails) { + return support.runExport(requestDetails, this); + } + + @Override + @Nonnull + public PreAsyncValidationResult preAsyncValidate( + @Nonnull final ServletRequestDetails servletRequestDetails, @Nonnull final Object[] params) + throws InvalidRequestException { + final SqlQueryExportRequest request = + requestParser.parse( + servletRequestDetails, + /* boundLibrary= */ null, + support.stringParam(servletRequestDetails, "_format"), + support.headerParam(servletRequestDetails), + support.stringParam(servletRequestDetails, "clientTrackingId"), + support.collectPatientIds(servletRequestDetails), + support.sinceParam(servletRequestDetails), + support.stringParam(servletRequestDetails, "source")); + return new PreAsyncValidationResult<>(request, Collections.emptyList()); + } + + @Override + @Nonnull + public String computeCacheKeyComponent(@Nonnull final SqlQueryExportRequest request) { + return support.computeCacheKeyComponent(request); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequest.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequest.java new file mode 100644 index 0000000000..0ec130d74c --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequest.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.operations.view.ViewExportFormat; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.Set; +import org.hl7.fhir.r4.model.InstantType; + +/** + * The parsed and validated kick-off request for a {@code $sqlquery-export} operation, produced by + * {@link SqlQueryExportRequestParser#parse} and carried to background execution via the job. Each + * {@link QueryInput} is already parsed, parameter bound, and view resolved; view sources are folded + * into the per-query resolved views and produce no outputs of their own. + * + * @param originalRequest the original request URL + * @param serverBaseUrl the FHIR server base URL (used for constructing result/download URLs) + * @param queries the ordered list of queries; one output per query + * @param clientTrackingId optional client-provided tracking identifier, echoed when present + * @param format the output format (NDJSON, CSV, or Parquet) + * @param includeHeader whether to include a header row in CSV output + * @param patientIds patient ids to filter by (from {@code patient} and {@code group} parameters) + * @param since filter resources modified after this timestamp + * @author John Grimes + */ +public record SqlQueryExportRequest( + @Nonnull String originalRequest, + @Nonnull String serverBaseUrl, + @Nonnull List queries, + @Nullable String clientTrackingId, + @Nonnull ViewExportFormat format, + boolean includeHeader, + @Nonnull Set patientIds, + @Nullable InstantType since) {} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java new file mode 100644 index 0000000000..82e7dd57b7 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java @@ -0,0 +1,350 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.errors.UnsupportedFhirPathFeatureError; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.view.ViewExecutionHelper; +import au.csiro.pathling.operations.view.ViewExportFormat; +import au.csiro.pathling.security.PathlingAuthority; +import au.csiro.pathling.security.ResourceAccess.AccessType; +import au.csiro.pathling.security.SecurityAspect; +import au.csiro.pathling.views.FhirView; +import au.csiro.pathling.views.FhirViewExecutor; +import au.csiro.pathling.views.ViewDefinitionGson; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import jakarta.validation.ConstraintViolationException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; +import org.hl7.fhir.r4.model.Reference; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Parses the raw {@code $sqlquery-export} kick-off inputs into a validated {@link + * SqlQueryExportRequest}. Performs every check that does not require executing the queries: source + * rejection, strict {@code _format} parsing, per-{@code query} and per-{@code view} exclusivity, + * query Library resolution, request-supplied view resolution and semantic validation, parameter + * binding, and static SQL validation. Each query is prepared (parsed, bound, view-resolved) via the + * shared {@link SqlQueryPipeline} so that the export and run operations share identical semantics. + * + * @author John Grimes + */ +@Component +public class SqlQueryExportRequestParser { + + @Nonnull private final SqlQueryPipeline pipeline; + + @Nonnull private final LibraryReferenceResolver libraryReferenceResolver; + + @Nonnull private final ViewExecutionHelper viewExecutionHelper; + + @Nonnull private final FhirContext fhirContext; + + @Nonnull private final ServerConfiguration serverConfiguration; + + @Nonnull private final QueryableDataSource deltaLake; + + @Nonnull private final Gson gson; + + /** + * Constructs a new SqlQueryExportRequestParser. + * + * @param pipeline the shared SQL query pipeline (prepare and static validation) + * @param libraryReferenceResolver resolves a {@code queryReference} to a stored SQLQuery Library + * @param viewExecutionHelper resolves a {@code view.viewReference} to a stored ViewDefinition, + * reusing the per-part exclusivity and reference-resolution semantics of the view operations + * @param fhirContext the FHIR context, used to serialise supplied ViewDefinitions for parsing + * @param serverConfiguration the server configuration (auth toggle and query config) + * @param deltaLake the data source used to semantically validate supplied ViewDefinitions + */ + @SuppressWarnings("java:S107") + @Autowired + public SqlQueryExportRequestParser( + @Nonnull final SqlQueryPipeline pipeline, + @Nonnull final LibraryReferenceResolver libraryReferenceResolver, + @Nonnull final ViewExecutionHelper viewExecutionHelper, + @Nonnull final FhirContext fhirContext, + @Nonnull final ServerConfiguration serverConfiguration, + @Nonnull final QueryableDataSource deltaLake) { + this.pipeline = pipeline; + this.libraryReferenceResolver = libraryReferenceResolver; + this.viewExecutionHelper = viewExecutionHelper; + this.fhirContext = fhirContext; + this.serverConfiguration = serverConfiguration; + this.deltaLake = deltaLake; + this.gson = ViewDefinitionGson.create(); + } + + /** + * Parses and validates the kick-off request. + * + * @param requestDetails the servlet request details (for the raw Parameters body and URLs) + * @param boundLibrary the bound Library at instance level, or null at system/type level + * @param format the explicit {@code _format} parameter, if any + * @param includeHeader whether to include a CSV header row; {@code null} defaults to {@code true} + * @param clientTrackingId optional client-provided tracking identifier + * @param patientIds patient ids to filter by, resolved from {@code patient} and {@code group} + * @param since the {@code _since} filter, if any + * @param source the unsupported {@code source} parameter, rejected when supplied + * @return the validated request + * @throws InvalidRequestException (400) for statically detectable structural failures + */ + @Nonnull + @SuppressWarnings("java:S107") + public SqlQueryExportRequest parse( + @Nonnull final ServletRequestDetails requestDetails, + @Nullable final IBaseResource boundLibrary, + @Nullable final String format, + @Nullable final BooleanType includeHeader, + @Nullable final String clientTrackingId, + @Nonnull final Set patientIds, + @Nullable final InstantType since, + @Nullable final String source) { + + // Reject the unsupported source parameter synchronously, before any other work. + if (source != null && !source.isBlank()) { + throw new InvalidRequestException( + "The 'source' parameter (external data source) is not supported by this server."); + } + + // Parse the explicit _format strictly, so an unsupported value (e.g. json, fhir) is rejected at + // kick-off regardless of the query parameters. + final ViewExportFormat exportFormat = ViewExportFormat.fromString(format); + + final Parameters parameters = extractParameters(requestDetails); + + // Resolve request-supplied views (system/type level only); the bound-Library instance level + // resolves its views from server storage. + final Map suppliedViews = + boundLibrary == null ? resolveSuppliedViews(parameters) : Map.of(); + + final List queries = new ArrayList<>(); + if (boundLibrary != null) { + // Instance level: the bound Library is the single query source; the query parameter does not + // apply and per-query parameter binding is not offered. + queries.add(prepareQuery(null, boundLibrary, null, suppliedViews)); + } else { + for (final RawQuery rawQuery : extractQueries(parameters)) { + final IBaseResource library = + selectLibrary(rawQuery.queryResource(), rawQuery.queryReference()); + queries.add(prepareQuery(rawQuery.name(), library, rawQuery.parameters(), suppliedViews)); + } + if (queries.isEmpty()) { + throw new InvalidRequestException( + "At least one 'query' parameter is required at the system and type levels."); + } + } + + final boolean header = includeHeader == null || includeHeader.booleanValue(); + + return new SqlQueryExportRequest( + requestDetails.getCompleteUrl(), + requestDetails.getFhirServerBase(), + queries, + clientTrackingId, + exportFormat, + header, + patientIds, + since); + } + + /** + * Prepares a single query: resolves its Library name fallback, then parses, binds parameters, and + * resolves views via the shared pipeline, and statically validates the SQL. + */ + @Nonnull + private QueryInput prepareQuery( + @Nullable final String name, + @Nonnull final IBaseResource library, + @Nullable final Parameters parameters, + @Nonnull final Map suppliedViews) { + final PreparedSqlQuery prepared = + pipeline.prepare(library, null, null, null, null, parameters, suppliedViews); + pipeline.validateStatically(prepared); + return new QueryInput(name, libraryName(library), prepared); + } + + /** Returns the SQLQuery Library's {@code name} element, or null when not a named Library. */ + @Nullable + private static String libraryName(@Nonnull final IBaseResource library) { + return library instanceof final Library lib && lib.hasName() ? lib.getName() : null; + } + + /** + * Enforces the "exactly one of queryResource / queryReference" contract and resolves the Library. + */ + @Nonnull + private IBaseResource selectLibrary( + @Nullable final IBaseResource queryResource, @Nullable final Reference queryReference) { + final boolean hasResource = queryResource != null; + final boolean hasReference = queryReference != null && !queryReference.isEmpty(); + + if (hasResource && hasReference) { + throw new InvalidRequestException( + "Each 'query' must supply exactly one of 'queryResource' and 'queryReference', not" + + " both."); + } + if (!hasResource && !hasReference) { + throw new InvalidRequestException( + "Each 'query' must supply one of 'queryResource' or 'queryReference'."); + } + return hasResource ? queryResource : libraryReferenceResolver.resolve(queryReference); + } + + /** + * Resolves the {@code view} parts into a map keyed by the ViewDefinition id they satisfy, parsing + * inline views, reading referenced views, applying the per-resource READ check to stored views, + * and semantically validating each supplied view (a malformed view is a 400; a semantically + * invalid one a 422). + */ + @Nonnull + private Map resolveSuppliedViews(@Nonnull final Parameters parameters) { + final Map resolved = new LinkedHashMap<>(); + for (final ParametersParameterComponent param : parameters.getParameter()) { + if (!"view".equals(param.getName())) { + continue; + } + IBaseResource viewResource = null; + Reference viewReference = null; + for (final ParametersParameterComponent part : param.getPart()) { + if ("viewResource".equals(part.getName()) && part.getResource() != null) { + viewResource = part.getResource(); + } else if ("viewReference".equals(part.getName()) + && part.getValue() instanceof final Reference reference) { + viewReference = reference; + } + } + + // resolveViewInput enforces per-part exclusivity (400), presence (400), and reference + // resolution (404), and returns the resolved ViewDefinition resource. + final boolean inline = viewResource != null; + final IBaseResource resolvedResource = + viewExecutionHelper.resolveViewInput(viewResource, viewReference); + final FhirView view = parseViewDefinition(resolvedResource); + + // A stored ViewDefinition is subject to the per-resource READ check; an inline view carries + // its own content and is authorised as the request payload. + if (!inline && serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority( + PathlingAuthority.resourceAccess(AccessType.READ, view.getResource())); + } + + validateViewSemantically(view); + + final String matchKey = resolvedResource.getIdElement().getIdPart(); + if (matchKey != null && !matchKey.isBlank()) { + resolved.put(matchKey, view); + } + } + return resolved; + } + + /** Parses a ViewDefinition resource into a FhirView via JSON round-tripping. */ + @Nonnull + private FhirView parseViewDefinition(@Nonnull final IBaseResource viewResource) { + try { + final String viewJson = fhirContext.newJsonParser().encodeResourceToString(viewResource); + return gson.fromJson(viewJson, FhirView.class); + } catch (final JsonSyntaxException e) { + throw new InvalidRequestException("Invalid ViewDefinition: " + e.getMessage()); + } + } + + /** + * Semantically validates a supplied ViewDefinition by building its query plan, consistent with + * the view operations: a semantically invalid view is a 422, an unsupported expression a 400. + */ + private void validateViewSemantically(@Nonnull final FhirView view) { + try { + new FhirViewExecutor(fhirContext, deltaLake, serverConfiguration.getQuery()).buildQuery(view); + } catch (final ConstraintViolationException e) { + throw new UnprocessableEntityException("Invalid ViewDefinition: " + e.getMessage()); + } catch (final UnsupportedOperationException | UnsupportedFhirPathFeatureError e) { + throw new InvalidRequestException("Unsupported expression: " + e.getMessage()); + } + } + + /** Extracts the raw {@code query} parts from the request Parameters, preserving order. */ + @Nonnull + private List extractQueries(@Nonnull final Parameters parameters) { + final List queries = new ArrayList<>(); + for (final ParametersParameterComponent param : parameters.getParameter()) { + if (!"query".equals(param.getName())) { + continue; + } + String name = null; + IBaseResource queryResource = null; + Reference queryReference = null; + Parameters queryParameters = null; + for (final ParametersParameterComponent part : param.getPart()) { + switch (part.getName()) { + case "name" -> name = part.getValue() != null ? part.getValue().primitiveValue() : null; + case "queryResource" -> queryResource = part.getResource(); + case "queryReference" -> { + if (part.getValue() instanceof final Reference reference) { + queryReference = reference; + } + } + case "parameters" -> { + if (part.getResource() instanceof final Parameters params) { + queryParameters = params; + } + } + default -> { + // Ignore unrecognised parts. + } + } + } + queries.add(new RawQuery(name, queryResource, queryReference, queryParameters)); + } + return queries; + } + + /** Extracts the Parameters resource from the request body, or an empty one when absent. */ + @Nonnull + private static Parameters extractParameters(@Nonnull final ServletRequestDetails requestDetails) { + return requestDetails.getResource() instanceof final Parameters parameters + ? parameters + : new Parameters(); + } + + /** A raw, unresolved {@code query} part extracted from the request Parameters. */ + private record RawQuery( + @Nullable String name, + @Nullable IBaseResource queryResource, + @Nullable Reference queryReference, + @Nullable Parameters parameters) {} +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java new file mode 100644 index 0000000000..3c8c7086b8 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java @@ -0,0 +1,349 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static au.csiro.pathling.security.SecurityAspect.getCurrentUserId; + +import au.csiro.pathling.async.AsyncJobContext; +import au.csiro.pathling.async.Job; +import au.csiro.pathling.async.JobRegistry; +import au.csiro.pathling.async.PreAsyncValidation; +import au.csiro.pathling.async.PreAsyncValidation.PreAsyncValidationResult; +import au.csiro.pathling.async.RequestTag; +import au.csiro.pathling.async.RequestTagFactory; +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.errors.AccessDeniedError; +import au.csiro.pathling.operations.bulkexport.ExportResult; +import au.csiro.pathling.operations.bulkexport.ExportResultRegistry; +import au.csiro.pathling.operations.compartment.GroupMemberService; +import au.csiro.pathling.operations.export.ExportFileWriter; +import au.csiro.pathling.operations.export.ExportManifest; +import au.csiro.pathling.operations.export.ExportManifestOutput; +import au.csiro.pathling.views.ViewDefinitionGson; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import com.google.gson.Gson; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.Type; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +/** + * Shared machinery for the {@code $sqlquery-export} providers across the system, type, and instance + * levels: owning-job resolution, background execution, completion-manifest construction, the + * deterministic cache key, and extraction of the simple kick-off parameters from the request body. + * Both {@link SqlQueryExportProvider} (system) and {@link SqlQueryInstanceExportProvider} + * (type/instance) delegate here so the level-agnostic logic lives in one place. + * + * @author John Grimes + */ +@Component +public class SqlQueryExportSupport { + + @Nonnull private final SqlQueryExportExecutor executor; + + @Nonnull private final JobRegistry jobRegistry; + + @Nonnull private final RequestTagFactory requestTagFactory; + + @Nonnull private final ExportResultRegistry exportResultRegistry; + + @Nonnull private final ServerConfiguration serverConfiguration; + + @Nonnull private final GroupMemberService groupMemberService; + + @Nonnull private final ExportFileWriter fileWriter; + + @Nonnull private final Gson gson; + + /** + * Constructs a new SqlQueryExportSupport. + * + * @param executor runs the queries and writes the output files + * @param jobRegistry the async job registry + * @param requestTagFactory the request tag factory used for job deduplication + * @param exportResultRegistry the export result registry backing the {@code $result} endpoint + * @param serverConfiguration the server configuration + * @param groupMemberService resolves {@code group} references to member patient ids + * @param fileWriter the shared export file writer, used to clean up partial outputs on failure + */ + @SuppressWarnings("java:S107") + @Autowired + public SqlQueryExportSupport( + @Nonnull final SqlQueryExportExecutor executor, + @Nonnull final JobRegistry jobRegistry, + @Nonnull final RequestTagFactory requestTagFactory, + @Nonnull final ExportResultRegistry exportResultRegistry, + @Nonnull final ServerConfiguration serverConfiguration, + @Nonnull final GroupMemberService groupMemberService, + @Nonnull final ExportFileWriter fileWriter) { + this.executor = executor; + this.jobRegistry = jobRegistry; + this.requestTagFactory = requestTagFactory; + this.exportResultRegistry = exportResultRegistry; + this.serverConfiguration = serverConfiguration; + this.groupMemberService = groupMemberService; + this.fileWriter = fileWriter; + this.gson = ViewDefinitionGson.create(); + } + + /** + * Resolves the owning job, runs the export in the background, and builds the completion manifest. + * + * @param requestDetails the request details + * @param validation the provider's pre-async validation (used for the fallback job lookup) + * @return the completion manifest, or null if the job was cancelled + */ + @Nullable + public Parameters runExport( + @Nonnull final ServletRequestDetails requestDetails, + @Nonnull final PreAsyncValidation validation) { + final Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + + final Job ownJob = + resolveOwnJob(requestDetails, authentication, validation); + if (ownJob == null) { + throw new InvalidRequestException("Missing 'Prefer: respond-async' header value."); + } + + // Check that the user requesting the result is the same user that started the job. + final Optional currentUserId = getCurrentUserId(authentication); + if (currentUserId.isPresent() && !ownJob.getOwnerId().equals(currentUserId)) { + throw new AccessDeniedError( + "The requested result is not owned by the current user '%s'." + .formatted(currentUserId.orElse("null"))); + } + + final SqlQueryExportRequest exportRequest = ownJob.getPreAsyncValidationResult(); + if (ownJob.isCancelled()) { + return null; + } + + exportResultRegistry.put(ownJob.getId(), new ExportResult(ownJob.getOwnerId())); + + final List outputs; + try { + outputs = executor.execute(exportRequest, ownJob.getId()); + } catch (final RuntimeException e) { + // All-or-nothing: a failed query fails the whole export. Remove the result registration and + // delete any partial outputs so none are offered for download, then surface the failure. + exportResultRegistry.remove(ownJob.getId()); + fileWriter.deleteJobDirectory(ownJob.getId()); + throw e; + } + + // Set the Expires header on the completion response. + ownJob.setResponseModification( + httpServletResponse -> { + final String expiresValue = + ZonedDateTime.now(ZoneOffset.UTC) + .plusSeconds(serverConfiguration.getExport().getResultExpiry()) + .format(DateTimeFormatter.RFC_1123_DATE_TIME); + httpServletResponse.addHeader("Expires", expiresValue); + }); + + return new ExportManifest( + exportRequest.serverBaseUrl(), + ownJob.getId(), + exportRequest.clientTrackingId(), + exportRequest.format().getCode(), + ownJob.getStartTime(), + Instant.now(), + outputs) + .toParameters(); + } + + /** + * Resolves the job owning this request: the one set by the async aspect when running + * asynchronously, or - as a fallback when the async context is unavailable - the one looked up by + * recomputing the request tag. + */ + @Nullable + private Job resolveOwnJob( + @Nonnull final ServletRequestDetails requestDetails, + @Nullable final Authentication authentication, + @Nonnull final PreAsyncValidation validation) { + @SuppressWarnings("unchecked") + final Optional> contextJob = + AsyncJobContext.getCurrentJob().map(job -> (Job) job); + if (contextJob.isPresent()) { + return contextJob.get(); + } + + final PreAsyncValidationResult validationResult = + validation.preAsyncValidate(requestDetails, new Object[] {}); + final String operationCacheKey = + validation.computeCacheKeyComponent( + Objects.requireNonNull( + validationResult.result(), + "Validation result should not be null for a valid request")); + final RequestTag ownTag = + requestTagFactory.createTag(requestDetails, authentication, operationCacheKey); + return jobRegistry.get(ownTag); + } + + /** + * Computes the deterministic cache key component from the parsed request, so that identical + * kick-offs deduplicate to the same job. + * + * @param request the parsed request + * @return the cache key component + */ + @Nonnull + public String computeCacheKeyComponent(@Nonnull final SqlQueryExportRequest request) { + final StringBuilder key = new StringBuilder(); + + final String queriesJson = + request.queries().stream() + .map( + q -> + (q.name() != null ? q.name() : "") + + ":" + + q.preparedQuery().getRequest().getParsedQuery().getSql() + + ":" + + gson.toJson(q.preparedQuery().getResolvedViews()) + + ":" + + gson.toJson(q.preparedQuery().getRequest().getParameterBindings())) + .collect(Collectors.joining(",")); + key.append("queries=[").append(queriesJson).append("]"); + + if (request.clientTrackingId() != null) { + key.append("|clientTrackingId=").append(request.clientTrackingId()); + } + key.append("|format=").append(request.format()); + key.append("|header=").append(request.includeHeader()); + + if (!request.patientIds().isEmpty()) { + final String sortedPatientIds = + request.patientIds().stream().sorted().collect(Collectors.joining(",")); + key.append("|patientIds=[").append(sortedPatientIds).append("]"); + } + if (request.since() != null) { + key.append("|since=").append(request.since().getValueAsString()); + } + return key.toString(); + } + + /** Collects patient ids from both the {@code patient} and {@code group} parameters. */ + @Nonnull + public Set collectPatientIds(@Nonnull final ServletRequestDetails requestDetails) { + final Set allPatientIds = new HashSet<>(); + for (final Parameters.ParametersParameterComponent param : + parametersOf(requestDetails).getParameter()) { + if ("patient".equals(param.getName())) { + final String id = stripResourcePrefix(referenceOrPrimitive(param.getValue())); + if (id != null && !id.isBlank()) { + allPatientIds.add(id); + } + } else if ("group".equals(param.getName())) { + final String groupId = stripResourcePrefix(referenceOrPrimitive(param.getValue())); + if (groupId != null && !groupId.isBlank()) { + allPatientIds.addAll(groupMemberService.extractPatientIdsFromGroup(groupId)); + } + } + } + return allPatientIds; + } + + /** + * Extracts a simple string-valued parameter from the request body. + * + * @param requestDetails the request details + * @param name the parameter name + * @return the primitive value, or null when absent + */ + @Nullable + public String stringParam( + @Nonnull final ServletRequestDetails requestDetails, @Nonnull final String name) { + for (final Parameters.ParametersParameterComponent param : + parametersOf(requestDetails).getParameter()) { + if (name.equals(param.getName()) && param.getValue() != null) { + return param.getValue().primitiveValue(); + } + } + return null; + } + + /** + * Extracts the {@code header} boolean parameter from the request body. + * + * @param requestDetails the request details + * @return the header flag, or null when absent + */ + @Nullable + public BooleanType headerParam(@Nonnull final ServletRequestDetails requestDetails) { + final String value = stringParam(requestDetails, "header"); + return value == null ? null : new BooleanType(value); + } + + /** + * Extracts the {@code _since} instant parameter from the request body. + * + * @param requestDetails the request details + * @return the since instant, or null when absent + */ + @Nullable + public InstantType sinceParam(@Nonnull final ServletRequestDetails requestDetails) { + final String value = stringParam(requestDetails, "_since"); + return value == null ? null : new InstantType(value); + } + + @Nullable + private static String referenceOrPrimitive(@Nullable final Type value) { + if (value == null) { + return null; + } + if (value instanceof final Reference reference) { + return reference.getReference(); + } + return value.primitiveValue(); + } + + @Nullable + private static String stripResourcePrefix(@Nullable final String reference) { + if (reference == null) { + return null; + } + final int slash = reference.lastIndexOf('/'); + return slash >= 0 ? reference.substring(slash + 1) : reference; + } + + @Nonnull + private static Parameters parametersOf(@Nonnull final ServletRequestDetails requestDetails) { + return requestDetails.getResource() instanceof final Parameters parameters + ? parameters + : new Parameters(); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java new file mode 100644 index 0000000000..7296256372 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java @@ -0,0 +1,196 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.async.AsyncSupported; +import au.csiro.pathling.async.PreAsyncValidation; +import au.csiro.pathling.security.OperationAccess; +import ca.uhn.fhir.rest.annotation.IdParam; +import ca.uhn.fhir.rest.annotation.Operation; +import ca.uhn.fhir.rest.annotation.OperationParam; +import ca.uhn.fhir.rest.server.IResourceProvider; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Reference; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Provider for the type-level and instance-level {@code $sqlquery-export} operations for Library + * resources: + * + *

      + *
    • {@code POST /fhir/Library/$sqlquery-export} - type level, accepting the {@code query} and + * {@code view} parameters exactly as the system level does. + *
    • {@code POST /fhir/Library/[id]/$sqlquery-export} - instance level, exporting the bound + * Library as the single query source. + *
    + * + *

    Both reuse the shared {@link SqlQueryExportSupport} and {@link SqlQueryExportRequestParser}. + * + * @author John Grimes + * @see SqlQueryExportProvider for the system-level operation + */ +@Slf4j +@Component +public class SqlQueryInstanceExportProvider + implements IResourceProvider, PreAsyncValidation { + + @Nonnull private final SqlQueryExportRequestParser requestParser; + + @Nonnull private final SqlQueryExportSupport support; + + @Nonnull private final LibraryReferenceResolver libraryReferenceResolver; + + /** + * Constructs a new SqlQueryInstanceExportProvider. + * + * @param requestParser parses and validates the kick-off request + * @param support the shared export machinery + * @param libraryReferenceResolver resolves the bound Library id at instance level + */ + @Autowired + public SqlQueryInstanceExportProvider( + @Nonnull final SqlQueryExportRequestParser requestParser, + @Nonnull final SqlQueryExportSupport support, + @Nonnull final LibraryReferenceResolver libraryReferenceResolver) { + this.requestParser = requestParser; + this.support = support; + this.libraryReferenceResolver = libraryReferenceResolver; + } + + @Override + public Class getResourceType() { + return Library.class; + } + + /** + * Type-level {@code $sqlquery-export} operation, accepting the repeating {@code query} and {@code + * view} parameters exactly as the system-level operation does. + * + * @param clientTrackingId optional client-provided tracking identifier + * @param format the output format (ndjson, csv, parquet) + * @param includeHeader whether to include headers in CSV output + * @param patientIds patient ids to filter by + * @param groupIds group ids to filter by + * @param since filter resources modified after this timestamp + * @param source the unsupported external data source parameter, rejected when supplied + * @param requestDetails the request details + * @return the completion manifest, or null if cancelled + */ + @SuppressWarnings({"unused", "java:S107"}) + @Operation(name = "$sqlquery-export", idempotent = true) + @OperationAccess("sqlquery-export") + @AsyncSupported(redirectOnComplete = true) + @Nullable + public Parameters exportType( + @Nullable @OperationParam(name = "clientTrackingId") final String clientTrackingId, + @Nullable @OperationParam(name = "_format") final String format, + @Nullable @OperationParam(name = "header") final BooleanType includeHeader, + @Nullable @OperationParam(name = "patient") final List patientIds, + @Nullable @OperationParam(name = "group") final List groupIds, + @Nullable @OperationParam(name = "_since") final InstantType since, + @Nullable @OperationParam(name = "source") final String source, + @Nonnull final ServletRequestDetails requestDetails) { + return support.runExport(requestDetails, this); + } + + /** + * Instance-level {@code $sqlquery-export} operation, exporting the bound Library as the single + * query source. + * + * @param libraryId the id of the stored Library to export + * @param format the output format (ndjson, csv, parquet) + * @param includeHeader whether to include headers in CSV output + * @param patientIds patient ids to filter by + * @param groupIds group ids to filter by + * @param since filter resources modified after this timestamp + * @param source the unsupported external data source parameter, rejected when supplied + * @param requestDetails the request details + * @return the completion manifest, or null if cancelled + */ + @SuppressWarnings({"unused", "java:S107"}) + @Operation(name = "$sqlquery-export", idempotent = true) + @OperationAccess("sqlquery-export") + @AsyncSupported(redirectOnComplete = true) + @Nullable + public Parameters exportInstance( + @IdParam final IdType libraryId, + @Nullable @OperationParam(name = "_format") final String format, + @Nullable @OperationParam(name = "header") final BooleanType includeHeader, + @Nullable @OperationParam(name = "patient") final List patientIds, + @Nullable @OperationParam(name = "group") final List groupIds, + @Nullable @OperationParam(name = "_since") final InstantType since, + @Nullable @OperationParam(name = "source") final String source, + @Nonnull final ServletRequestDetails requestDetails) { + return support.runExport(requestDetails, this); + } + + @Override + @Nonnull + public PreAsyncValidationResult preAsyncValidate( + @Nonnull final ServletRequestDetails servletRequestDetails, @Nonnull final Object[] params) + throws InvalidRequestException { + // The bound Library at instance level is identified by the request id in the path; its absence + // marks a type-level invocation, which carries the query/view parameters instead. + final IBaseResource boundLibrary = resolveBoundLibrary(servletRequestDetails); + + final SqlQueryExportRequest request = + requestParser.parse( + servletRequestDetails, + boundLibrary, + support.stringParam(servletRequestDetails, "_format"), + support.headerParam(servletRequestDetails), + support.stringParam(servletRequestDetails, "clientTrackingId"), + support.collectPatientIds(servletRequestDetails), + support.sinceParam(servletRequestDetails), + support.stringParam(servletRequestDetails, "source")); + return new PreAsyncValidationResult<>(request, Collections.emptyList()); + } + + /** Resolves the bound Library at instance level, or returns null for a type-level invocation. */ + @Nullable + private IBaseResource resolveBoundLibrary( + @Nonnull final ServletRequestDetails servletRequestDetails) { + final IdType id = + servletRequestDetails.getId() == null + ? null + : new IdType(servletRequestDetails.getId().getValue()); + if (id == null || id.getIdPart() == null || id.getIdPart().isBlank()) { + return null; + } + return libraryReferenceResolver.resolve(new Reference("Library/" + id.getIdPart())); + } + + @Override + @Nonnull + public String computeCacheKeyComponent(@Nonnull final SqlQueryExportRequest request) { + return support.computeCacheKeyComponent(request); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java new file mode 100644 index 0000000000..d994f5bbe2 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java @@ -0,0 +1,148 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.io.source.DataSource; +import au.csiro.pathling.views.FhirView; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Parameters; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * The shared request/execution pipeline for SQL queries. Given a SQLQuery Library, optional runtime + * parameters, and optional request-supplied views, it parses the query, resolves its ViewDefinition + * table sources, statically validates the SQL, and executes it against Spark. + * + *

    Both the synchronous {@code $sqlquery-run} operation (which streams the single result) and the + * asynchronous {@code $sqlquery-export} operation (which writes each result to files) call this + * pipeline, so the parsing, view resolution, validation, and execution semantics are identical + * across the two. Only the terminal step (stream-to-response vs. write-to-files) differs and is + * supplied by the caller as a {@link Consumer} of the result dataset. + * + * @author John Grimes + */ +@Component +public class SqlQueryPipeline { + + @Nonnull private final SqlQueryRequestParser requestParser; + + @Nonnull private final ViewResolver viewResolver; + + @Nonnull private final SqlQueryExecutor executor; + + @Nonnull private final SqlValidator sqlValidator; + + /** + * Constructs a new SqlQueryPipeline. + * + * @param requestParser parses a SQLQuery Library and binds runtime parameters + * @param viewResolver resolves the view references to parsed FhirViews, preferring + * request-supplied views over server storage + * @param executor validates and runs the SQL against Spark + * @param sqlValidator the static SQL validator, used for kick-off-time validation that does not + * require executing the query + */ + @Autowired + public SqlQueryPipeline( + @Nonnull final SqlQueryRequestParser requestParser, + @Nonnull final ViewResolver viewResolver, + @Nonnull final SqlQueryExecutor executor, + @Nonnull final SqlValidator sqlValidator) { + this.requestParser = requestParser; + this.viewResolver = viewResolver; + this.executor = executor; + this.sqlValidator = sqlValidator; + } + + /** + * Parses the SQLQuery Library and resolves its view table sources, producing a {@link + * PreparedSqlQuery} ready for validation and execution. Performs all structural FHIR-level + * validation (query parsing, parameter binding and type checking) and view resolution (preferring + * request-supplied views, falling back to server storage), but does not touch Spark. + * + * @param library the SQLQuery Library resource (inline or already resolved from a reference) + * @param format the explicit {@code _format} parameter, if any + * @param acceptHeader the HTTP {@code Accept} header value, used as a fallback for {@code format} + * @param includeHeader whether to include a CSV header row; {@code null} defaults to {@code true} + * @param limit optional row cap + * @param parameters runtime parameter bindings as a {@code Parameters} resource + * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy + * @return the prepared query + */ + @Nonnull + @SuppressWarnings("java:S107") + public PreparedSqlQuery prepare( + @Nonnull final IBaseResource library, + @Nullable final String format, + @Nullable final String acceptHeader, + @Nullable final BooleanType includeHeader, + @Nullable final IntegerType limit, + @Nullable final Parameters parameters, + @Nonnull final Map suppliedViews) { + final SqlQueryRequest request = + requestParser.parse(library, format, acceptHeader, includeHeader, limit, parameters); + final Map resolvedViews = + viewResolver.resolve(request.getParsedQuery().getViewReferences(), suppliedViews); + return new PreparedSqlQuery(request, resolvedViews); + } + + /** + * Runs the static SQL validation that does not require executing the query, so that malformed or + * disallowed SQL is detected before any Spark work. Used by the asynchronous export to surface + * these failures synchronously at kick-off. + * + * @param prepared the prepared query + */ + public void validateStatically(@Nonnull final PreparedSqlQuery prepared) { + final Set declaredLabels = + prepared.getRequest().getParsedQuery().getViewReferences().stream() + .map(ViewArtifactReference::getLabel) + .collect(Collectors.toUnmodifiableSet()); + sqlValidator.validate(prepared.getRequest().getParsedQuery().getSql(), declaredLabels); + } + + /** + * Executes the prepared query against Spark, registering the resolved views under request-scoped + * temp views for the duration of the call and invoking {@code consumer} with the result dataset + * before they are dropped. + * + * @param prepared the prepared query + * @param dataSource the data source backing FhirView execution (filtered for the export filters) + * @param requestId the HAPI per-request id used to namespace temp view names + * @param consumer terminal consumer of the result dataset + */ + public void execute( + @Nonnull final PreparedSqlQuery prepared, + @Nonnull final DataSource dataSource, + @Nonnull final String requestId, + @Nonnull final Consumer> consumer) { + executor.execute( + prepared.getRequest(), prepared.getResolvedViews(), dataSource, requestId, consumer); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java index 556a08c2f2..b1355cb62d 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java @@ -74,7 +74,8 @@ public ViewResolver( } /** - * Resolves each view reference to a parsed {@link FhirView}, preserving label order. + * Resolves each view reference to a parsed {@link FhirView}, reading every view from server + * storage. Equivalent to {@link #resolve(List, Map)} with no request-supplied views. * * @param references the references declared by the SQLQuery Library, keyed by table label * @return a map from label to resolved FhirView @@ -82,10 +83,39 @@ public ViewResolver( */ @Nonnull public Map resolve(@Nonnull final List references) { + return resolve(references, Map.of()); + } + + /** + * Resolves each view reference to a parsed {@link FhirView}, preserving label order. A + * request-supplied view is used in preference to server storage when it matches the reference (by + * the ViewDefinition id extracted from the reference); otherwise the view is read from server + * storage, exactly as {@code $sqlquery-run} does. Request-supplied views are assumed already + * parsed and, for stored references, authorisation-checked by the caller. + * + * @param references the references declared by the SQLQuery Library, keyed by table label + * @param suppliedViews request-supplied views keyed by the ViewDefinition id (or canonical url's + * final segment) they satisfy; may be empty + * @return a map from label to resolved FhirView + * @throws InvalidRequestException if a reference cannot be resolved or parsed + */ + @Nonnull + public Map resolve( + @Nonnull final List references, + @Nonnull final Map suppliedViews) { final Map resolved = new LinkedHashMap<>(); for (final ViewArtifactReference ref : references) { final String viewDefinitionId = extractViewDefinitionId(ref.getCanonicalUrl()); + + // Prefer a request-supplied view that satisfies this reference, falling back to server + // storage when none is supplied. + final FhirView suppliedView = suppliedViews.get(viewDefinitionId); + if (suppliedView != null) { + resolved.put(ref.getLabel(), suppliedView); + continue; + } + final IBaseResource viewResource; try { viewResource = readExecutor.read("ViewDefinition", viewDefinitionId); diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutor.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutor.java index a369fb7d47..72308f79dd 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutor.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutor.java @@ -17,41 +17,31 @@ package au.csiro.pathling.operations.view; -import static au.csiro.pathling.library.io.FileSystemPersistence.safelyJoinPaths; - import au.csiro.pathling.config.ServerConfiguration; -import au.csiro.pathling.library.io.FileSystemPersistence; import au.csiro.pathling.library.io.source.QueryableDataSource; -import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import au.csiro.pathling.operations.export.ExportDataSourceBuilder; +import au.csiro.pathling.operations.export.ExportFileWriter; import au.csiro.pathling.views.FhirView; import au.csiro.pathling.views.FhirViewExecutor; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; import jakarta.validation.ConstraintViolationException; -import java.io.IOException; -import java.net.URI; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.spark.sql.AnalysisException; -import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; /** - * Executes ViewDefinition queries and writes the results to files. + * Executes ViewDefinition queries and writes the results to files, reusing the shared {@link + * ExportDataSourceBuilder} (filtering) and {@link ExportFileWriter} (job directory and file + * writing) that back both export operations. * * @author John Grimes */ @@ -63,39 +53,33 @@ public class ViewDefinitionExportExecutor { @Nonnull private final FhirContext fhirContext; - @Nonnull private final SparkSession sparkSession; - - @Nonnull private final String databasePath; - @Nonnull private final ServerConfiguration serverConfiguration; - @Nonnull private final PatientCompartmentService patientCompartmentService; + @Nonnull private final ExportDataSourceBuilder dataSourceBuilder; + + @Nonnull private final ExportFileWriter fileWriter; /** * Constructs a new ViewDefinitionExportExecutor. * * @param deltaLake the queryable data source * @param fhirContext the FHIR context - * @param sparkSession the Spark session - * @param databasePath the database path * @param serverConfiguration the server configuration - * @param patientCompartmentService the patient compartment service + * @param dataSourceBuilder the shared export data-source builder (applies filters) + * @param fileWriter the shared export file writer (job directory and file writing) */ @Autowired public ViewDefinitionExportExecutor( @Nonnull final QueryableDataSource deltaLake, @Nonnull final FhirContext fhirContext, - @Nonnull final SparkSession sparkSession, - @Nonnull @Value("${pathling.storage.warehouseUrl}/${pathling.storage.databaseName}") - final String databasePath, @Nonnull final ServerConfiguration serverConfiguration, - @Nonnull final PatientCompartmentService patientCompartmentService) { + @Nonnull final ExportDataSourceBuilder dataSourceBuilder, + @Nonnull final ExportFileWriter fileWriter) { this.deltaLake = deltaLake; this.fhirContext = fhirContext; - this.sparkSession = sparkSession; - this.databasePath = databasePath; this.serverConfiguration = serverConfiguration; - this.patientCompartmentService = patientCompartmentService; + this.dataSourceBuilder = dataSourceBuilder; + this.fileWriter = fileWriter; } /** @@ -109,74 +93,35 @@ public ViewDefinitionExportExecutor( public List execute( @Nonnull final ViewDefinitionExportRequest request, @Nonnull final String jobId) { - final Path jobDirPath = createJobDirectory(jobId); + final Path jobDirPath = fileWriter.createJobDirectory(jobId); + final QueryableDataSource dataSource = + dataSourceBuilder.build(deltaLake, request.since(), request.patientIds()); final List outputs = new ArrayList<>(); final Set usedNames = new HashSet<>(); for (int i = 0; i < request.views().size(); i++) { final ViewInput viewInput = request.views().get(i); - final String viewName = getUniqueViewName(viewInput.getEffectiveName(i), usedNames); + final String viewName = fileWriter.uniqueName(viewInput.getEffectiveName(i), usedNames); usedNames.add(viewName); - final ViewExportOutput output = executeView(viewInput.view(), viewName, request, jobDirPath); + final ViewExportOutput output = + executeView(viewInput.view(), viewName, request, dataSource, jobDirPath); outputs.add(output); } return outputs; } - /** Creates the job directory for storing output files. */ - @Nonnull - private Path createJobDirectory(@Nonnull final String jobId) { - final URI warehouseUri = URI.create(databasePath); - final Path warehousePath = new Path(warehouseUri); - final Path jobDirPath = new Path(new Path(warehousePath, "jobs"), jobId); - final Configuration configuration = sparkSession.sparkContext().hadoopConfiguration(); - - try { - final FileSystem fs = FileSystem.get(configuration); - if (!fs.exists(jobDirPath)) { - final boolean created = fs.mkdirs(jobDirPath); - if (!created) { - throw new InternalErrorException( - "Failed to create subdirectory at %s for job %s.".formatted(databasePath, jobId)); - } - log.debug("Created dir {}", jobDirPath); - } - } catch (final IOException e) { - throw new InternalErrorException( - "Failed to create subdirectory at %s for job %s.".formatted(databasePath, jobId)); - } - - return jobDirPath; - } - - /** Generates a unique name for a view, avoiding collisions with already-used names. */ - @Nonnull - private String getUniqueViewName( - @Nonnull final String baseName, @Nonnull final Set usedNames) { - if (!usedNames.contains(baseName)) { - return baseName; - } - int suffix = 1; - while (usedNames.contains(baseName + "_" + suffix)) { - suffix++; - } - return baseName + "_" + suffix; - } - /** Executes a single view and writes the results to files. */ @Nonnull private ViewExportOutput executeView( @Nonnull final FhirView view, @Nonnull final String viewName, @Nonnull final ViewDefinitionExportRequest request, + @Nonnull final QueryableDataSource dataSource, @Nonnull final Path jobDirPath) { - // Build the data source with filters applied. - final QueryableDataSource dataSource = buildDataSource(request); - - // Execute the view query. + // Execute the view query against the filtered data source. final FhirViewExecutor executor = new FhirViewExecutor(fhirContext, dataSource, serverConfiguration.getQuery()); @@ -188,60 +133,14 @@ private ViewExportOutput executeView( "Invalid ViewDefinition '%s': %s".formatted(viewName, e.getMessage())); } - // Write output based on format. + // Write the output in the requested format, reusing the shared file writer. final List fileUrls = writeOutput(result, viewName, request.format(), request.includeHeader(), jobDirPath); return new ViewExportOutput(viewName, fileUrls); } - /** Builds the data source with filters applied. */ - @Nonnull - private QueryableDataSource buildDataSource(@Nonnull final ViewDefinitionExportRequest request) { - QueryableDataSource dataSource = deltaLake; - - // Apply _since filter. - if (request.since() != null) { - dataSource = - dataSource.map( - rowDataset -> - rowDataset.filter( - "meta.lastUpdated IS NULL OR meta.lastUpdated >= '" - + request.since().getValueAsString() - + "'")); - } - - // Apply patient compartment filter if patient IDs were specified. - if (!request.patientIds().isEmpty()) { - dataSource = applyPatientCompartmentFilter(dataSource, request.patientIds()); - } - - return dataSource; - } - - /** Applies patient compartment filter to the data source. */ - @Nonnull - private QueryableDataSource applyPatientCompartmentFilter( - @Nonnull final QueryableDataSource dataSource, @Nonnull final Set patientIds) { - - // Filter out resource types that are not in the Patient compartment. - final QueryableDataSource filtered = - dataSource.filterByResourceType(patientCompartmentService::isInPatientCompartment); - - // Apply row-level filtering based on patient compartment membership. - return filtered.map( - (resourceType, rowDataset) -> { - final Column patientFilter = - patientCompartmentService.buildPatientFilter(resourceType, patientIds); - log.debug( - "Applying patient compartment filter for resource type {}: {}", - resourceType, - patientFilter); - return rowDataset.filter(patientFilter); - }); - } - - /** Writes the query result to files in the specified format. */ + /** Writes the query result in the requested format via the shared file writer. */ @Nonnull private List writeOutput( @Nonnull final Dataset result, @@ -249,88 +148,10 @@ private List writeOutput( @Nonnull final ViewExportFormat format, final boolean includeHeader, @Nonnull final Path jobDirPath) { - - final String outputPath = - safelyJoinPaths(jobDirPath.toString(), viewName + format.getFileExtension()); - - switch (format) { - case NDJSON -> { - return writeNdjson(result, outputPath, viewName, jobDirPath); - } - case CSV -> { - return writeCsv(result, outputPath, viewName, includeHeader, jobDirPath); - } - case PARQUET -> { - return writeParquet(result, outputPath, viewName, jobDirPath); - } - default -> throw new IllegalArgumentException("Unsupported format: " + format); - } - } - - /** Writes the result as NDJSON files. */ - @Nonnull - private List writeNdjson( - @Nonnull final Dataset result, - @Nonnull final String outputPath, - @Nonnull final String viewName, - @Nonnull final Path jobDirPath) { - - result.write().mode(SaveMode.Overwrite).json(outputPath); - - // Rename partitioned files to follow naming convention. - return new ArrayList<>( - FileSystemPersistence.renamePartitionedFiles(sparkSession, outputPath, outputPath, "json")); - } - - /** - * Writes the result as CSV files. - * - * @throws InvalidRequestException if the dataset contains data types that are not supported by - * the CSV format - */ - @Nonnull - private List writeCsv( - @Nonnull final Dataset result, - @Nonnull final String outputPath, - @Nonnull final String viewName, - final boolean includeHeader, - @Nonnull final Path jobDirPath) { - - try { - result.write().mode(SaveMode.Overwrite).option("header", includeHeader).csv(outputPath); - } catch (final Exception e) { - // Spark throws AnalysisException when it encounters unsupported data types for a datasource. - // We convert this to an InvalidRequestException to return a 400 status code. - if (e instanceof final AnalysisException ae - && "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE".equals(ae.getErrorClass())) { - throw new InvalidRequestException( - "CSV export failed for view '%s': %s".formatted(viewName, e.getMessage())); - } - // Re-throw unexpected exceptions as runtime exceptions. - if (e instanceof RuntimeException re) { - throw re; - } - throw new RuntimeException(e); - } - - // Rename partitioned files to follow naming convention. - return new ArrayList<>( - FileSystemPersistence.renamePartitionedFiles(sparkSession, outputPath, outputPath, "csv")); - } - - /** Writes the result as Parquet files. */ - @Nonnull - private List writeParquet( - @Nonnull final Dataset result, - @Nonnull final String outputPath, - @Nonnull final String viewName, - @Nonnull final Path jobDirPath) { - - result.write().mode(SaveMode.Overwrite).parquet(outputPath); - - // Rename partitioned files to follow the numbered naming convention. - return new ArrayList<>( - FileSystemPersistence.renamePartitionedFiles( - sparkSession, outputPath, outputPath, "parquet")); + return switch (format) { + case NDJSON -> fileWriter.writeNdjson(result, viewName, jobDirPath); + case CSV -> fileWriter.writeCsv(result, viewName, includeHeader, jobDirPath); + case PARQUET -> fileWriter.writeParquet(result, viewName, jobDirPath); + }; } } diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java index afbb20f25e..c32e7c323d 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportResponse.java @@ -18,23 +18,14 @@ package au.csiro.pathling.operations.view; import au.csiro.pathling.OperationResponse; -import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import au.csiro.pathling.operations.export.ExportManifest; +import au.csiro.pathling.operations.export.ExportManifestOutput; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; -import java.net.URISyntaxException; -import java.time.Duration; import java.time.Instant; -import java.util.Date; import java.util.List; import lombok.Getter; -import org.apache.http.client.utils.URIBuilder; -import org.hl7.fhir.r4.model.CodeType; -import org.hl7.fhir.r4.model.InstantType; -import org.hl7.fhir.r4.model.IntegerType; import org.hl7.fhir.r4.model.Parameters; -import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; -import org.hl7.fhir.r4.model.StringType; -import org.hl7.fhir.r4.model.UriType; /** * Represents the completion manifest returned at the result URL of a {@code $viewdefinition-export} @@ -99,79 +90,21 @@ public ViewDefinitionExportResponse( @Nonnull @Override public Parameters toOutput() { - final Parameters parameters = new Parameters(); - - // Ensure the base URL ends with a slash for proper URL construction. - final String normalizedBaseUrl = - serverBaseUrl.endsWith("/") ? serverBaseUrl : serverBaseUrl + "/"; - - parameters.addParameter().setName("exportId").setValue(new StringType(exportId)); - parameters.addParameter().setName("status").setValue(new CodeType("completed")); - - // Echo the client tracking id only when one was supplied at kick-off. - if (clientTrackingId != null && !clientTrackingId.isBlank()) { - parameters - .addParameter() - .setName("clientTrackingId") - .setValue(new StringType(clientTrackingId)); - } - - parameters.addParameter().setName("_format").setValue(new CodeType(format)); - - parameters - .addParameter() - .setName("exportStartTime") - .setValue(new InstantType(Date.from(exportStartTime))); - parameters - .addParameter() - .setName("exportEndTime") - .setValue(new InstantType(Date.from(exportEndTime))); - - // Whole seconds between start and end, never negative. - final long durationSeconds = - Math.max(0L, Duration.between(exportStartTime, exportEndTime).toSeconds()); - parameters - .addParameter() - .setName("exportDuration") - .setValue(new IntegerType((int) durationSeconds)); - - // One output per view, with a name and one or more location parts. - for (final ViewExportOutput output : outputs) { - final ParametersParameterComponent outputParam = parameters.addParameter().setName("output"); - outputParam.addPart().setName("name").setValue(new StringType(output.name())); - for (final String fileUrl : output.fileUrls()) { - outputParam - .addPart() - .setName("location") - .setValue(new UriType(buildResultUrl(normalizedBaseUrl, fileUrl))); - } - } - - return parameters; - } - - /** - * Converts a local file URL to a remote result URL. - * - * @param baseUrl the normalised base server URL - * @param localUrl the local file URL containing the job ID and filename - * @return the remote result URL - */ - @Nonnull - private static String buildResultUrl( - @Nonnull final String baseUrl, @Nonnull final String localUrl) { - try { - final String[] parts = localUrl.split("/jobs/")[1].split("/"); - final String jobUuid = parts[0]; - final String file = parts[1]; - - return new URIBuilder(baseUrl + "$result") - .addParameter("job", jobUuid) - .addParameter("file", file) - .build() - .toString(); - } catch (final URISyntaxException e) { - throw new InternalErrorException(e); - } + // Delegate to the shared manifest builder, mapping each view output to the generic export + // output shape. Both export operations produce the identical manifest, so the construction + // lives in one place. + final List manifestOutputs = + outputs.stream() + .map(output -> new ExportManifestOutput(output.name(), output.fileUrls())) + .toList(); + return new ExportManifest( + serverBaseUrl, + exportId, + clientTrackingId, + format, + exportStartTime, + exportEndTime, + manifestOutputs) + .toParameters(); } } diff --git a/server/src/main/resources/application.yml b/server/src/main/resources/application.yml index 7803ee930c..0e08a93861 100644 --- a/server/src/main/resources/application.yml +++ b/server/src/main/resources/application.yml @@ -85,6 +85,7 @@ pathling: viewDefinitionRunEnabled: true viewDefinitionInstanceRunEnabled: true viewDefinitionExportEnabled: true + sqlQueryExportEnabled: true bulkSubmitEnabled: true # Resource limits applied to the $sqlquery-run operation. Both limits are always applied, diff --git a/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java b/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java index 7a7ecefa57..fb87cfce74 100644 --- a/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java @@ -271,6 +271,27 @@ void sqlQueryRunDeclaresSpecCanonical() { .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$sqlquery-run"); } + @Test + void sqlQueryExportDeclaresSpecCanonical() { + final CapabilityStatement capabilityStatement = + conformanceProvider.getServerConformance(null, null); + assertThat(systemOperationDefinition(capabilityStatement, "sqlquery-export")) + .isEqualTo("http://sql-on-fhir.org/OperationDefinition/$sqlquery-export"); + } + + @Test + void sqlQueryExportNotDeclaredWhenDisabled() { + final ConformanceProvider provider = + createProviderWithDisabledOperations(ops -> ops.setSqlQueryExportEnabled(false)); + final CapabilityStatement capabilityStatement = provider.getServerConformance(null, null); + + final Set operationNames = + capabilityStatement.getRest().getFirst().getOperation().stream() + .map(CapabilityStatementRestResourceOperationComponent::getName) + .collect(Collectors.toSet()); + assertThat(operationNames).doesNotContain("sqlquery-export"); + } + @Test void authoredSqlOnFhirOperationDefinitionsNoLongerServed() { for (final String name : diff --git a/server/src/test/java/au/csiro/pathling/operations/export/ExportManifestTest.java b/server/src/test/java/au/csiro/pathling/operations/export/ExportManifestTest.java new file mode 100644 index 0000000000..db96b07206 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/export/ExportManifestTest.java @@ -0,0 +1,222 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.export; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.List; +import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.UriType; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the shared {@link ExportManifest} builder, verifying the SQL on FHIR completion + * manifest shape produced by both export operations. + * + * @author John Grimes + */ +class ExportManifestTest { + + private static final String BASE_URL = "http://example.org/fhir"; + private static final Instant START = Instant.parse("2026-06-21T01:00:00Z"); + private static final Instant END = Instant.parse("2026-06-21T01:00:12Z"); + + @Test + void manifestContainsExportIdAndCompletedStatus() { + final Parameters parameters = manifest("job-123", "tracking-1", "ndjson", List.of()); + + assertThat(getStringParameter(parameters, "exportId")).isEqualTo("job-123"); + assertThat(getStringParameter(parameters, "status")).isEqualTo("completed"); + } + + @Test + void manifestEchoesEffectiveFormat() { + final Parameters parameters = manifest("job-123", null, "csv", List.of()); + assertThat(getStringParameter(parameters, "_format")).isEqualTo("csv"); + } + + @Test + void manifestEchoesClientTrackingIdWhenSupplied() { + final Parameters parameters = manifest("job-123", "tracking-abc", "ndjson", List.of()); + assertThat(getStringParameter(parameters, "clientTrackingId")).isEqualTo("tracking-abc"); + } + + @Test + void manifestOmitsClientTrackingIdWhenAbsent() { + final Parameters parameters = manifest("job-123", null, "ndjson", List.of()); + assertThat(hasParameter(parameters, "clientTrackingId")).isFalse(); + } + + @Test + void manifestContainsTimingFields() { + final Parameters parameters = manifest("job-123", null, "ndjson", List.of()); + + assertThat(findParameter(parameters, "exportStartTime").getValue()) + .isInstanceOf(InstantType.class); + assertThat(findParameter(parameters, "exportEndTime").getValue()) + .isInstanceOf(InstantType.class); + + final ParametersParameterComponent duration = findParameter(parameters, "exportDuration"); + assertThat(duration.getValue()).isInstanceOf(IntegerType.class); + assertThat(((IntegerType) duration.getValue()).getValue()).isEqualTo(12); + } + + @Test + void manifestOmitsCancelUrlAndEstimatedTimeRemaining() { + final ExportManifestOutput output = + new ExportManifestOutput( + "patients", List.of("file:///tmp/jobs/abc-123/patients.ndjson/part-00000.json")); + final Parameters parameters = manifest("abc-123", "t", "ndjson", List.of(output)); + + assertThat(hasParameter(parameters, "cancelUrl")).isFalse(); + assertThat(hasParameter(parameters, "estimatedTimeRemaining")).isFalse(); + } + + @Test + void outputHasNameAndLocationPartsMappedToResultUrl() { + final ExportManifestOutput output = + new ExportManifestOutput( + "patients", List.of("file:///tmp/jobs/abc-123/patients.ndjson/part-00000.json")); + final Parameters parameters = manifest("abc-123", null, "ndjson", List.of(output)); + + final List outputs = getParametersByName(parameters, "output"); + assertThat(outputs).hasSize(1); + + final ParametersParameterComponent outputParam = outputs.get(0); + assertThat(getPartValue(outputParam, "name")).isEqualTo("patients"); + assertThat(getPartValue(outputParam, "location")).contains("$result").contains("job=abc-123"); + assertThat(hasPart(outputParam, "url")).isFalse(); + } + + @Test + void partitionedOutputRepeatsLocationOncePerFile() { + final ExportManifestOutput output = + new ExportManifestOutput( + "observations", + List.of( + "file:///tmp/jobs/job-id/observations.ndjson/part-00000.json", + "file:///tmp/jobs/job-id/observations.ndjson/part-00001.json")); + final Parameters parameters = manifest("job-id", null, "ndjson", List.of(output)); + + final List outputs = getParametersByName(parameters, "output"); + assertThat(outputs).hasSize(1); + final List locations = + outputs.get(0).getPart().stream().filter(p -> "location".equals(p.getName())).toList(); + assertThat(locations).hasSize(2); + } + + @Test + void oneOutputPerUnit() { + final ExportManifestOutput a = + new ExportManifestOutput("a", List.of("file:///tmp/jobs/job-id/a.csv/part-00000.csv")); + final ExportManifestOutput b = + new ExportManifestOutput("b", List.of("file:///tmp/jobs/job-id/b.csv/part-00000.csv")); + final Parameters parameters = manifest("job-id", null, "csv", List.of(a, b)); + + final List outputs = getParametersByName(parameters, "output"); + assertThat(outputs).hasSize(2); + assertThat(getPartValue(outputs.get(0), "name")).isEqualTo("a"); + assertThat(getPartValue(outputs.get(1), "name")).isEqualTo("b"); + } + + @Test + void noOutputParametersWhenNoOutputs() { + final Parameters parameters = manifest("job-id", null, "ndjson", List.of()); + assertThat(getParametersByName(parameters, "output")).isEmpty(); + } + + @Test + void baseUrlWithTrailingSlashDoesNotProduceDoubleSlash() { + final ExportManifestOutput output = + new ExportManifestOutput( + "test", List.of("file:///tmp/jobs/job-id/test.ndjson/part-00000.json")); + final Parameters parameters = + new ExportManifest(BASE_URL + "/", "job-id", null, "ndjson", START, END, List.of(output)) + .toParameters(); + + final String location = + getPartValue(getParametersByName(parameters, "output").get(0), "location"); + assertThat(location).startsWith("http://example.org/fhir/$result"); + assertThat(location).doesNotContain("fhir//$result"); + } + + // ------------------------------------------------------------------------- + // Helper methods + // ------------------------------------------------------------------------- + + private static Parameters manifest( + final String exportId, + final String clientTrackingId, + final String format, + final List outputs) { + return new ExportManifest(BASE_URL, exportId, clientTrackingId, format, START, END, outputs) + .toParameters(); + } + + private static boolean hasParameter(final Parameters parameters, final String name) { + return parameters.getParameter().stream().anyMatch(p -> name.equals(p.getName())); + } + + private static ParametersParameterComponent findParameter( + final Parameters parameters, final String name) { + return parameters.getParameter().stream() + .filter(p -> name.equals(p.getName())) + .findFirst() + .orElse(null); + } + + private static List getParametersByName( + final Parameters parameters, final String name) { + return parameters.getParameter().stream().filter(p -> name.equals(p.getName())).toList(); + } + + private static String getStringParameter(final Parameters parameters, final String name) { + final ParametersParameterComponent param = findParameter(parameters, name); + if (param == null || !param.hasValue()) { + return null; + } + return param.getValue().primitiveValue(); + } + + private static boolean hasPart(final ParametersParameterComponent param, final String partName) { + return param.getPart().stream().anyMatch(p -> partName.equals(p.getName())); + } + + private static String getPartValue( + final ParametersParameterComponent param, final String partName) { + return param.getPart().stream() + .filter(p -> partName.equals(p.getName())) + .findFirst() + .map( + p -> { + if (p.getValue() instanceof final UriType uriType) { + return uriType.getValue(); + } + if (p.getValue() instanceof final StringType stringType) { + return stringType.getValue(); + } + return p.getValue().primitiveValue(); + }) + .orElse(null); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/AbstractSqlQueryExportIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/AbstractSqlQueryExportIT.java new file mode 100644 index 0000000000..c7bed03ebe --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/AbstractSqlQueryExportIT.java @@ -0,0 +1,363 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.gson.Gson; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.http.HttpStatus; +import org.springframework.test.web.reactive.server.WebTestClient; + +/** + * Shared infrastructure for the {@code $sqlquery-export} integration tests: the async kick-off, + * polling, result retrieval, and download helpers, plus the Parameters request builders and parsing + * helpers. Concrete subclasses supply the {@code @SpringBootTest} configuration and the test + * methods. + * + * @author John Grimes + */ +abstract class AbstractSqlQueryExportIT { + + @LocalServerPort protected int port; + + @Autowired protected WebTestClient webTestClient; + + protected final Gson gson = new Gson(); + + @BeforeEach + void setUpClient() { + webTestClient = + webTestClient + .mutate() + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(100 * 1024 * 1024)) + .build(); + } + + // ------------------------------------------------------------------------- + // Endpoints + // ------------------------------------------------------------------------- + + @Nonnull + protected String systemLevelUri() { + return "http://localhost:" + port + "/fhir/$sqlquery-export"; + } + + @Nonnull + protected String typeLevelUri() { + return "http://localhost:" + port + "/fhir/Library/$sqlquery-export"; + } + + @Nonnull + protected String instanceLevelUri(@Nonnull final String libraryId) { + return "http://localhost:" + port + "/fhir/Library/" + libraryId + "/$sqlquery-export"; + } + + // ------------------------------------------------------------------------- + // Async flow + // ------------------------------------------------------------------------- + + @Nonnull + protected WebTestClient.ResponseSpec kickOff( + @Nonnull final String uri, @Nonnull final Map body) { + return webTestClient + .post() + .uri(uri) + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/fhir+json") + .header("Prefer", "respond-async") + .bodyValue(gson.toJson(body)) + .exchange(); + } + + @Nonnull + protected Map exportToCompletion( + @Nonnull final String uri, @Nonnull final Map body) + throws InterruptedException { + final byte[] manifest = + webTestClient + .get() + .uri(resultLocationOf(uri, body)) + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult() + .getResponseBodyContent(); + return parse(manifest); + } + + /** Kicks off an export, polls to completion, and returns the result URL from the 303 redirect. */ + @Nonnull + protected String resultLocationOf( + @Nonnull final String uri, @Nonnull final Map body) + throws InterruptedException { + final String contentLocation = + kickOff(uri, body) + .expectStatus() + .isAccepted() + .returnResult(String.class) + .getResponseHeaders() + .getFirst("Content-Location"); + assertThat(contentLocation).isNotNull(); + + String resultLocation = null; + for (int attempt = 0; attempt < 60 && resultLocation == null; attempt++) { + final var poll = + webTestClient + .get() + .uri(contentLocation) + .header("Accept", "application/fhir+json") + .exchange() + .returnResult(String.class); + final HttpStatus status = (HttpStatus) poll.getStatus(); + if (status == HttpStatus.SEE_OTHER) { + resultLocation = poll.getResponseHeaders().getFirst("Location"); + } else if (status == HttpStatus.ACCEPTED) { + Thread.sleep(500); + } else { + throw new AssertionError("Unexpected poll status: " + status); + } + } + assertThat(resultLocation).as("Expected a 303 result location within the timeout").isNotNull(); + return resultLocation; + } + + /** Kicks off an export expected to fail, polling until the result URL returns an error status. */ + @Nonnull + protected WebTestClient.ResponseSpec resultOfFailedExport( + @Nonnull final String uri, @Nonnull final Map body) + throws InterruptedException { + final String contentLocation = + kickOff(uri, body) + .expectStatus() + .isAccepted() + .returnResult(String.class) + .getResponseHeaders() + .getFirst("Content-Location"); + assertThat(contentLocation).isNotNull(); + + for (int attempt = 0; attempt < 60; attempt++) { + final var poll = + webTestClient + .get() + .uri(contentLocation) + .header("Accept", "application/fhir+json") + .exchange() + .returnResult(String.class); + final HttpStatus status = (HttpStatus) poll.getStatus(); + if (status == HttpStatus.SEE_OTHER) { + final String resultLocation = poll.getResponseHeaders().getFirst("Location"); + return webTestClient + .get() + .uri(resultLocation) + .header("Accept", "application/fhir+json") + .exchange(); + } else if (status == HttpStatus.ACCEPTED) { + Thread.sleep(500); + } else { + // The failure surfaced directly on the status poll. + return webTestClient + .get() + .uri(contentLocation) + .header("Accept", "application/fhir+json") + .exchange(); + } + } + throw new AssertionError("Export neither completed nor failed within the timeout"); + } + + @Nonnull + protected byte[] downloadBytes(@Nonnull final String location) { + final byte[] bytes = + webTestClient + .get() + .uri(location) + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult() + .getResponseBodyContent(); + return bytes == null ? new byte[0] : bytes; + } + + @Nonnull + protected String download(@Nonnull final String location) { + return new String(downloadBytes(location), StandardCharsets.UTF_8); + } + + // ------------------------------------------------------------------------- + // Request builders + // ------------------------------------------------------------------------- + + @Nonnull + protected Map storedQuery( + @Nonnull final String libraryId, @Nullable final String format) { + final Map parameters = parametersWith(queryPartWithReference(libraryId)); + if (format != null) { + addSimpleParam(parameters, "_format", "valueString", format); + } + return parameters; + } + + @Nonnull + protected Map queryPartWithReference(@Nonnull final String libraryId) { + final Map queryParam = new LinkedHashMap<>(); + queryParam.put("name", "query"); + queryParam.put( + "part", new ArrayList<>(List.of(referencePart("queryReference", "Library/" + libraryId)))); + return queryParam; + } + + @Nonnull + protected Map parametersWith(@Nonnull final Map param) { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + parameters.put("parameter", new ArrayList<>(List.of(param))); + return parameters; + } + + @Nonnull + protected Map emptyParameters() { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + parameters.put("parameter", new ArrayList<>()); + return parameters; + } + + @SuppressWarnings("unchecked") + protected void addParam( + @Nonnull final Map parameters, @Nonnull final Map param) { + ((List>) parameters.get("parameter")).add(param); + } + + @SuppressWarnings("unchecked") + protected void addSimpleParam( + @Nonnull final Map parameters, + @Nonnull final String name, + @Nonnull final String valueKey, + @Nonnull final Object value) { + final Map part = new LinkedHashMap<>(); + part.put("name", name); + part.put(valueKey, value); + ((List>) parameters.get("parameter")).add(part); + } + + @Nonnull + protected Map referencePart( + @Nonnull final String name, @Nonnull final String reference) { + final Map part = new LinkedHashMap<>(); + part.put("name", name); + part.put("valueReference", Map.of("reference", reference)); + return part; + } + + @Nonnull + protected Map resourcePart( + @Nonnull final String name, @Nonnull final Map resource) { + final Map part = new LinkedHashMap<>(); + part.put("name", name); + part.put("resource", resource); + return part; + } + + // ------------------------------------------------------------------------- + // Parameters parsing + // ------------------------------------------------------------------------- + + @SuppressWarnings("unchecked") + @Nonnull + protected Map parse(@Nullable final byte[] body) { + return gson.fromJson( + new String(body == null ? new byte[0] : body, StandardCharsets.UTF_8), Map.class); + } + + @SuppressWarnings("unchecked") + @Nullable + protected static Map findParam( + @Nonnull final Map parameters, @Nonnull final String name) { + final List> list = (List>) parameters.get("parameter"); + if (list == null) { + return null; + } + return list.stream().filter(p -> name.equals(p.get("name"))).findFirst().orElse(null); + } + + @Nullable + protected static String findParamValue( + @Nonnull final Map parameters, + @Nonnull final String name, + @Nonnull final String valueKey) { + final Map param = findParam(parameters, name); + return param == null ? null : (String) param.get(valueKey); + } + + @SuppressWarnings("unchecked") + @Nonnull + protected static List> paramsByName( + @Nonnull final Map parameters, @Nonnull final String name) { + final List> list = (List>) parameters.get("parameter"); + final List> result = new ArrayList<>(); + if (list != null) { + for (final Map p : list) { + if (name.equals(p.get("name"))) { + result.add(p); + } + } + } + return result; + } + + @SuppressWarnings("unchecked") + @Nonnull + protected static List partValues( + @Nonnull final Map param, + @Nonnull final String partName, + @Nonnull final String valueKey) { + final List> parts = (List>) param.get("part"); + final List values = new ArrayList<>(); + if (parts != null) { + for (final Map p : parts) { + if (partName.equals(p.get("name"))) { + values.add((String) p.get(valueKey)); + } + } + } + return values; + } + + @Nullable + protected static String partValue( + @Nonnull final Map param, + @Nonnull final String partName, + @Nonnull final String valueKey) { + final List values = partValues(param, partName, valueKey); + return values.isEmpty() ? null : values.get(0); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java new file mode 100644 index 0000000000..f99003b997 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java @@ -0,0 +1,135 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.config.AuthorizationConfiguration; +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; +import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; +import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.views.FhirView; +import ca.uhn.fhir.context.FhirContext; +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.Map; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the request-supplied view resolution path of {@link ViewResolver}: a supplied view + * is matched to a {@code relatedArtifact} reference by id and preferred over server storage, while + * a reference with no matching supplied view falls back to storage. + * + * @author John Grimes + */ +class RequestViewResolutionTest { + + private ReadExecutor readExecutor; + private ViewResolver resolver; + + @BeforeEach + void setUp() { + readExecutor = mock(ReadExecutor.class); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + final AuthorizationConfiguration auth = new AuthorizationConfiguration(); + auth.setEnabled(false); + serverConfiguration.setAuth(auth); + resolver = new ViewResolver(readExecutor, serverConfiguration, FhirContext.forR4()); + } + + @Test + void suppliedViewIsPreferredAndStorageIsNotConsulted() { + final FhirView supplied = + FhirView.ofResource("Patient") + .select(FhirView.columns(FhirView.column("id", "id"))) + .build(); + final Map suppliedViews = Map.of("patient-bp", supplied); + + final Map resolved = + resolver.resolve( + List.of(new ViewArtifactReference("patients", "ViewDefinition/patient-bp")), + suppliedViews); + + assertThat(resolved.get("patients")).isSameAs(supplied); + verify(readExecutor, never()) + .read( + org.mockito.ArgumentMatchers.eq("ViewDefinition"), + org.mockito.ArgumentMatchers.anyString()); + } + + @Test + void referenceWithNoSuppliedViewFallsBackToStorage() { + when(readExecutor.read("ViewDefinition", "patient-bp")) + .thenReturn(simpleViewDefinition("patient-bp", "Patient")); + + final Map resolved = + resolver.resolve( + List.of(new ViewArtifactReference("patients", "ViewDefinition/patient-bp")), Map.of()); + + assertThat(resolved).containsOnlyKeys("patients"); + verify(readExecutor).read("ViewDefinition", "patient-bp"); + } + + @Test + void mixesSuppliedAndStorageResolvedViews() { + final FhirView supplied = + FhirView.ofResource("Patient") + .select(FhirView.columns(FhirView.column("id", "id"))) + .build(); + when(readExecutor.read("ViewDefinition", "obs-view")) + .thenReturn(simpleViewDefinition("obs-view", "Observation")); + + final Map resolved = + resolver.resolve( + List.of( + new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), + new ViewArtifactReference("obs", "ViewDefinition/obs-view")), + Map.of("patient-bp", supplied)); + + assertThat(resolved.get("patients")).isSameAs(supplied); + assertThat(resolved.get("obs").getResource()).isEqualTo("Observation"); + verify(readExecutor).read("ViewDefinition", "obs-view"); + verify(readExecutor, never()).read("ViewDefinition", "patient-bp"); + } + + @Nonnull + private static ViewDefinitionResource simpleViewDefinition( + @Nonnull final String id, @Nonnull final String resourceType) { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(id); + view.setName(new StringType(id + "_view")); + view.setResource(new CodeType(resourceType)); + view.setStatus(new CodeType("active")); + final SelectComponent select = new SelectComponent(); + final ColumnComponent column = new ColumnComponent(); + column.setName(new StringType("id")); + column.setPath(new StringType("id")); + select.getColumn().add(column); + view.getSelect().add(select); + return view; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportConformanceIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportConformanceIT.java new file mode 100644 index 0000000000..77b104a743 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportConformanceIT.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.web.reactive.server.WebTestClient; + +/** + * Integration test confirming the CapabilityStatement declares {@code $sqlquery-export} against the + * SQL on FHIR spec canonical when the operation is enabled (the default). + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +class SqlQueryExportConformanceIT { + + @LocalServerPort int port; + + @Autowired WebTestClient webTestClient; + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + } + + @Test + void capabilityStatementDeclaresSqlQueryExportAgainstSpecCanonical() { + final byte[] body = + webTestClient + .get() + .uri("http://localhost:" + port + "/fhir/metadata") + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult() + .getResponseBodyContent(); + + final String metadata = new String(body == null ? new byte[0] : body, StandardCharsets.UTF_8); + assertThat(metadata).contains("sqlquery-export"); + assertThat(metadata).contains("http://sql-on-fhir.org/OperationDefinition/$sqlquery-export"); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportDisabledIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportDisabledIT.java new file mode 100644 index 0000000000..489b99f091 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportDisabledIT.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +/** + * Integration test confirming that {@code $sqlquery-export} is not served when disabled by + * configuration (FR-033), and is absent from the CapabilityStatement. + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +class SqlQueryExportDisabledIT extends AbstractSqlQueryExportIT { + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + registry.add("pathling.operations.sqlQueryExportEnabled", () -> false); + } + + @Test + void operationIsNotServedWhenDisabled() { + // The operation is not registered, so HAPI rejects the kick-off as an unknown operation + // (a 4xx client error) rather than accepting it for processing. + kickOff(systemLevelUri(), storedQuery("any-library", null)).expectStatus().is4xxClientError(); + } + + @Test + void capabilityStatementDoesNotDeclareTheOperationWhenDisabled() { + final byte[] body = + webTestClient + .get() + .uri("http://localhost:" + port + "/fhir/metadata") + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult() + .getResponseBodyContent(); + + final String metadata = new String(body == null ? new byte[0] : body, StandardCharsets.UTF_8); + assertThat(metadata).doesNotContain("sqlquery-export"); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java new file mode 100644 index 0000000000..6f14c70622 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java @@ -0,0 +1,182 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.export.ExportDataSourceBuilder; +import au.csiro.pathling.operations.export.ExportFileWriter; +import au.csiro.pathling.operations.export.ExportManifestOutput; +import au.csiro.pathling.operations.view.ViewExportFormat; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import org.apache.hadoop.fs.Path; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link SqlQueryExportExecutor}, verifying that it produces one named output per + * query and writes via the shared file writer. The Spark-backed collaborators are mocked. + * + * @author John Grimes + */ +class SqlQueryExportExecutorTest { + + private SqlQueryPipeline pipeline; + private QueryableDataSource deltaLake; + private ExportDataSourceBuilder dataSourceBuilder; + private ExportFileWriter fileWriter; + private SqlQueryExportExecutor executor; + private Dataset dataset; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + pipeline = mock(SqlQueryPipeline.class); + deltaLake = mock(QueryableDataSource.class); + dataSourceBuilder = mock(ExportDataSourceBuilder.class); + fileWriter = mock(ExportFileWriter.class); + executor = new SqlQueryExportExecutor(pipeline, deltaLake, dataSourceBuilder, fileWriter); + dataset = mock(Dataset.class); + + when(dataSourceBuilder.build(any(), any(), anySet())).thenReturn(deltaLake); + when(fileWriter.createJobDirectory(any())).thenReturn(new Path("file:///tmp/jobs/job-1")); + // uniqueName returns the requested base name unchanged for these single-output cases. + when(fileWriter.uniqueName(any(), anySet())) + .thenAnswer(invocation -> invocation.getArgument(0)); + // The pipeline invokes the terminal consumer with the dataset, as the real one does. + doAnswer( + invocation -> { + final Consumer> consumer = invocation.getArgument(3); + consumer.accept(dataset); + return null; + }) + .when(pipeline) + .execute(any(), any(), any(), any()); + } + + @Test + void singleQueryYieldsOneNamedOutputWithFileUrls() { + final List fileUrls = List.of("file:///tmp/jobs/job-1/patients.ndjson/part-00000.json"); + when(fileWriter.writeNdjson(eq(dataset), eq("patients"), any())).thenReturn(fileUrls); + + final SqlQueryExportRequest request = request(ViewExportFormat.NDJSON, queryInput("patients")); + + final List outputs = executor.execute(request, "job-1"); + + assertThat(outputs).hasSize(1); + assertThat(outputs.get(0).name()).isEqualTo("patients"); + assertThat(outputs.get(0).fileUrls()).isEqualTo(fileUrls); + } + + @Test + void csvFormatRoutesToCsvWriterWithHeaderFlag() { + final List fileUrls = List.of("file:///tmp/jobs/job-1/patients.csv/part-00000.csv"); + when(fileWriter.writeCsv(eq(dataset), eq("patients"), eq(false), any())).thenReturn(fileUrls); + + final SqlQueryExportRequest request = + new SqlQueryExportRequest( + "http://x/$sqlquery-export", + "http://x/fhir", + List.of(queryInput("patients")), + null, + ViewExportFormat.CSV, + /* includeHeader= */ false, + java.util.Set.of(), + null); + + final List outputs = executor.execute(request, "job-1"); + + assertThat(outputs).hasSize(1); + assertThat(outputs.get(0).fileUrls()).isEqualTo(fileUrls); + } + + @Test + void partitionedResultYieldsOneOutputWithAllFileUrls() { + // A query partitioned into multiple files produces a single output that carries every file URL; + // the manifest builder then repeats the location part once per file (FR-028). + final List fileUrls = + List.of( + "file:///tmp/jobs/job-1/people.00000.ndjson", + "file:///tmp/jobs/job-1/people.00001.ndjson"); + when(fileWriter.writeNdjson(eq(dataset), eq("people"), any())).thenReturn(fileUrls); + + final List outputs = + executor.execute(request(ViewExportFormat.NDJSON, queryInput("people")), "job-1"); + + assertThat(outputs).hasSize(1); + assertThat(outputs.get(0).fileUrls()).containsExactlyElementsOf(fileUrls); + } + + @Test + void outputNameFollowsPrecedenceQueryNameThenLibraryNameThenGenerated() { + final PreparedSqlQuery prepared = + new PreparedSqlQuery( + new SqlQueryRequest( + new ParsedSqlQuery("SELECT 1", List.of(), List.of()), + SqlQueryOutputFormat.NDJSON, + true, + null, + Map.of()), + Map.of()); + + // query.name wins. + assertThat(new QueryInput("explicit", "lib", prepared).getEffectiveName(0)) + .isEqualTo("explicit"); + // Library name is the next fallback. + assertThat(new QueryInput(null, "lib_name", prepared).getEffectiveName(0)) + .isEqualTo("lib_name"); + // A generated name is the final fallback. + assertThat(new QueryInput(null, null, prepared).getEffectiveName(2)).isEqualTo("query_2"); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private static SqlQueryExportRequest request( + final ViewExportFormat format, final QueryInput... queries) { + return new SqlQueryExportRequest( + "http://x/$sqlquery-export", + "http://x/fhir", + List.of(queries), + null, + format, + true, + java.util.Set.of(), + null); + } + + private static QueryInput queryInput(final String name) { + final ParsedSqlQuery parsedQuery = + new ParsedSqlQuery("SELECT id FROM patients", List.of(), List.of()); + final SqlQueryRequest request = + new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); + return new QueryInput(name, null, new PreparedSqlQuery(request, Map.of())); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java new file mode 100644 index 0000000000..385e4e6820 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java @@ -0,0 +1,260 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.annotation.Nonnull; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +/** + * Integration tests for {@code $sqlquery-export} output formats, the CSV header toggle, filter + * scoping, the synchronous rejections (unsupported {@code _format} and {@code source}), and the + * all-or-nothing multi-query failure behaviour. + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +@Import(SqlQueryExportTestConfiguration.class) +class SqlQueryExportFormatIT extends AbstractSqlQueryExportIT { + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + } + + // ------------------------------------------------------------------------- + // Formats and CSV header + // ------------------------------------------------------------------------- + + @Test + void defaultFormatIsNdjson() throws InterruptedException { + final Map manifest = + exportToCompletion( + systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)); + final String location = + partValue(paramsByName(manifest, "output").get(0), "location", "valueUri"); + assertThat(location).endsWith(".ndjson"); + assertThat(findParamValue(manifest, "_format", "valueCode")).isEqualTo("ndjson"); + } + + @Test + void csvFormatIncludesHeaderByDefault() throws InterruptedException { + final Map manifest = + exportToCompletion( + systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, "csv")); + final String location = + partValue(paramsByName(manifest, "output").get(0), "location", "valueUri"); + final String content = download(location); + assertThat(content).startsWith("id,family_name"); + } + + @Test + void csvFormatOmitsHeaderWhenHeaderFalse() throws InterruptedException { + final Map body = + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, "csv"); + addSimpleParam(body, "header", "valueBoolean", Boolean.FALSE); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).doesNotContain("id,family_name"); + assertThat(content).contains("Smith"); + } + + @Test + void parquetFormatProducesParquetFiles() throws InterruptedException { + final Map manifest = + exportToCompletion( + systemLevelUri(), + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, "parquet")); + final String location = + partValue(paramsByName(manifest, "output").get(0), "location", "valueUri"); + assertThat(location).endsWith(".parquet"); + assertThat(findParamValue(manifest, "_format", "valueCode")).isEqualTo("parquet"); + } + + // ------------------------------------------------------------------------- + // Synchronous rejections + // ------------------------------------------------------------------------- + + @Test + void jsonFormatRejectedAtKickOff() { + kickOff(systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, "json")) + .expectStatus() + .isBadRequest(); + } + + @Test + void fhirFormatRejectedAtKickOff() { + kickOff(systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, "fhir")) + .expectStatus() + .isBadRequest(); + } + + @Test + void sourceParameterRejectedAtKickOff() { + final Map body = + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null); + addSimpleParam(body, "source", "valueString", "s3://bucket/data"); + + final byte[] outcome = + kickOff(systemLevelUri(), body) + .expectStatus() + .isBadRequest() + .expectBody() + .returnResult() + .getResponseBodyContent(); + assertThat(new String(outcome == null ? new byte[0] : outcome, StandardCharsets.UTF_8)) + .contains("source"); + } + + // ------------------------------------------------------------------------- + // Filtering + // ------------------------------------------------------------------------- + + @Test + void patientFilterScopesExportedRows() throws InterruptedException { + final Map body = + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null); + addParam(body, referencePart("patient", "Patient/p1")); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).contains("\"id\":\"p1\""); + assertThat(content).doesNotContain("\"id\":\"p2\"").doesNotContain("\"id\":\"p3\""); + } + + @Test + void groupFilterScopesExportedRows() throws InterruptedException { + final Map body = + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null); + // The group's sole member is p1, so only p1 should be exported. + addParam(body, referencePart("group", "Group/" + SqlQueryExportTestConfiguration.GROUP_ID)); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).contains("\"id\":\"p1\""); + assertThat(content).doesNotContain("\"id\":\"p2\"").doesNotContain("\"id\":\"p3\""); + } + + @Test + void sinceFilterScopesExportedRows() throws InterruptedException { + final Map body = + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null); + // p1 was last updated in 2020; p2 and p3 in 2025. A _since of 2023 excludes p1. + addSimpleParam(body, "_since", "valueInstant", "2023-01-01T00:00:00.000Z"); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).doesNotContain("\"id\":\"p1\""); + assertThat(content).contains("\"id\":\"p2\"").contains("\"id\":\"p3\""); + } + + // ------------------------------------------------------------------------- + // All-or-nothing multi-query failure + // ------------------------------------------------------------------------- + + @Test + void failingQueryFailsTheWholeExportWithNoManifest() throws InterruptedException { + final Map body = new LinkedHashMap<>(); + body.put("resourceType", "Parameters"); + final Map goodQuery = + queryPartWithReference(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID); + final Map badQuery = new LinkedHashMap<>(); + badQuery.put("name", "query"); + badQuery.put("part", new ArrayList<>(List.of(resourcePart("queryResource", failingLibrary())))); + body.put("parameter", new ArrayList<>(List.of(goodQuery, badQuery))); + + final byte[] result = + resultOfFailedExport(systemLevelUri(), body) + .expectStatus() + .value(status -> assertThat(status).isGreaterThanOrEqualTo(400)) + .expectBody() + .returnResult() + .getResponseBodyContent(); + + final Map parsed = parse(result); + // The failure is reported as an OperationOutcome, not a completion manifest. + assertThat(parsed.get("resourceType")).isEqualTo("OperationOutcome"); + assertThat(paramsByName(parsed, "output")).isEmpty(); + } + + /** An inline SQLQuery Library that parses but fails at execution (unresolved column). */ + @Nonnull + private Map failingLibrary() { + final Map library = new LinkedHashMap<>(); + library.put("resourceType", "Library"); + library.put("status", "active"); + library.put( + "type", + Map.of( + "coding", + List.of( + Map.of( + "system", + SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM, + "code", + SqlQueryLibraryParser.LIBRARY_TYPE_CODE)))); + final String sql = "SELECT nonexistent_column FROM patients"; + library.put( + "content", + List.of( + Map.of( + "contentType", + "application/sql", + "data", + java.util.Base64.getEncoder() + .encodeToString(sql.getBytes(StandardCharsets.UTF_8))))); + library.put( + "relatedArtifact", + List.of( + Map.of( + "type", + "depends-on", + "label", + "patients", + "resource", + "ViewDefinition/" + SqlQueryExportTestConfiguration.PATIENT_VIEW_ID))); + return library; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportInstanceIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportInstanceIT.java new file mode 100644 index 0000000000..57496e69b8 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportInstanceIT.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +/** + * Integration tests for the type-level and instance-level {@code $sqlquery-export} operations. + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +@Import(SqlQueryExportTestConfiguration.class) +class SqlQueryExportInstanceIT extends AbstractSqlQueryExportIT { + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + } + + @Test + void instanceLevelExportProducesOneOutputForBoundLibrary() throws InterruptedException { + final Map manifest = + exportToCompletion( + instanceLevelUri(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID), emptyParameters()); + + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("completed"); + assertThat(paramsByName(manifest, "output")).hasSize(1); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).contains("\"family_name\":\"Smith\""); + } + + @Test + void instanceLevelExportForNonExistentLibraryReturns404() { + kickOff(instanceLevelUri("does-not-exist"), emptyParameters()).expectStatus().isNotFound(); + } + + @Test + void typeLevelExportWithQueryProducesOutput() throws InterruptedException { + final Map manifest = + exportToCompletion( + typeLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)); + + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("completed"); + assertThat(paramsByName(manifest, "output")).hasSize(1); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java new file mode 100644 index 0000000000..6996486079 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java @@ -0,0 +1,402 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.annotation.Nonnull; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.http.HttpStatus; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +/** + * Integration tests for the system-level {@code $sqlquery-export} asynchronous flow: stored and + * inline queries, request-supplied views, multiple queries, and the manifest shape, lifetime, and + * cancellation. + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +@Import(SqlQueryExportTestConfiguration.class) +class SqlQueryExportProviderIT extends AbstractSqlQueryExportIT { + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + } + + // ------------------------------------------------------------------------- + // US1 - stored single-query export, full async flow + // ------------------------------------------------------------------------- + + @Test + void kickOffReturns202WithAcceptedAcknowledgement() { + final byte[] body = + kickOff( + systemLevelUri(), + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)) + .expectStatus() + .isAccepted() + .expectHeader() + .exists("Content-Location") + .expectBody() + .returnResult() + .getResponseBodyContent(); + + final Map ack = parse(body); + assertThat(ack.get("resourceType")).isEqualTo("Parameters"); + assertThat(findParamValue(ack, "status", "valueCode")).isEqualTo("accepted"); + assertThat(findParamValue(ack, "exportId", "valueString")).isNotNull(); + } + + @Test + void storedQueryExportsToNdjsonEndToEnd() throws InterruptedException { + final Map manifest = + exportToCompletion( + systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)); + + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("completed"); + final List> outputs = paramsByName(manifest, "output"); + assertThat(outputs).hasSize(1); + final String location = partValue(outputs.get(0), "location", "valueUri"); + assertThat(location).contains("$result"); + + final String content = download(location); + final String[] lines = content.trim().split("\n"); + assertThat(lines).hasSize(3); + assertThat(content) + .contains("\"family_name\":\"Smith\"") + .contains("\"family_name\":\"Johnson\"") + .contains("\"family_name\":\"Williams\""); + } + + @Test + void echoesClientTrackingIdInAcknowledgementAndManifest() throws InterruptedException { + final Map body = + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null); + addSimpleParam(body, "clientTrackingId", "valueString", "track-1"); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + assertThat(findParamValue(manifest, "clientTrackingId", "valueString")).isEqualTo("track-1"); + } + + @Test + void noQueryAtSystemLevelRejectedWith400() { + kickOff(systemLevelUri(), emptyParameters()).expectStatus().isBadRequest(); + } + + @Test + void queryWithBothSourcesRejectedWith400() { + final Map queryPart = + queryPartWithReference(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID); + @SuppressWarnings("unchecked") + final List> parts = (List>) queryPart.get("part"); + parts.add(resourcePart("queryResource", inlineSqlLibrary())); + + kickOff(systemLevelUri(), parametersWith(queryPart)).expectStatus().isBadRequest(); + } + + @Test + void queryReferenceToMissingLibraryRejectedWith404() { + kickOff(systemLevelUri(), storedQuery("does-not-exist", null)).expectStatus().isNotFound(); + } + + // ------------------------------------------------------------------------- + // US2 - inline query and request-supplied views + // ------------------------------------------------------------------------- + + @Test + void inlineQueryWithInlineViewExportsRows() throws InterruptedException { + final Map body = parametersWith(queryPartWithResource(inlineSqlLibrary())); + addParam(body, viewPartWithResource(inlineViewDefinition())); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + final List> outputs = paramsByName(manifest, "output"); + assertThat(outputs).hasSize(1); + final String content = download(partValue(outputs.get(0), "location", "valueUri")); + assertThat(content).contains("\"family_name\":\"Smith\""); + } + + @Test + void inlineQueryWithViewReferenceExportsRows() throws InterruptedException { + final Map body = parametersWith(queryPartWithResource(inlineSqlLibrary())); + addParam( + body, + viewPartWithReference("ViewDefinition/" + SqlQueryExportTestConfiguration.PATIENT_VIEW_ID)); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + assertThat(paramsByName(manifest, "output")).hasSize(1); + } + + @Test + void viewPartWithBothSourcesRejectedWith400() { + final Map viewPart = viewPartWithResource(inlineViewDefinition()); + @SuppressWarnings("unchecked") + final List> parts = (List>) viewPart.get("part"); + parts.add( + referencePart( + "viewReference", "ViewDefinition/" + SqlQueryExportTestConfiguration.PATIENT_VIEW_ID)); + + final Map body = parametersWith(queryPartWithResource(inlineSqlLibrary())); + addParam(body, viewPart); + kickOff(systemLevelUri(), body).expectStatus().isBadRequest(); + } + + @Test + void viewPartWithNeitherSourceRejectedWith400() { + final Map viewPart = new LinkedHashMap<>(); + viewPart.put("name", "view"); + viewPart.put("part", new ArrayList<>(List.of(simplePart("name", "valueString", "empty-view")))); + + final Map body = parametersWith(queryPartWithResource(inlineSqlLibrary())); + addParam(body, viewPart); + kickOff(systemLevelUri(), body).expectStatus().isBadRequest(); + } + + @Test + void semanticallyInvalidSuppliedViewRejectedWith422() { + final Map invalidView = inlineViewDefinition(); + invalidView.put("select", new ArrayList<>()); // Empty select is semantically invalid. + + final Map body = parametersWith(queryPartWithResource(inlineSqlLibrary())); + addParam(body, viewPartWithResource(invalidView)); + kickOff(systemLevelUri(), body).expectStatus().isEqualTo(HttpStatus.UNPROCESSABLE_ENTITY); + } + + // ------------------------------------------------------------------------- + // US3 - multiple queries in one export + // ------------------------------------------------------------------------- + + @Test + void twoQueriesProduceTwoNamedOutputs() throws InterruptedException { + final Map body = new LinkedHashMap<>(); + body.put("resourceType", "Parameters"); + final Map q1 = + queryPartWithReference(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID); + addPartTo(q1, simplePart("name", "valueString", "people")); + final Map q2 = + queryPartWithReference(SqlQueryExportTestConfiguration.OBSERVATION_QUERY_ID); + body.put("parameter", new ArrayList<>(List.of(q1, q2))); + + final Map manifest = exportToCompletion(systemLevelUri(), body); + final List> outputs = paramsByName(manifest, "output"); + assertThat(outputs).hasSize(2); + assertThat(partValue(outputs.get(0), "name", "valueString")).isEqualTo("people"); + assertThat(partValue(outputs.get(1), "name", "valueString")) + .isEqualTo("observation_weight_query"); + } + + @Test + void perQueryParameterBindingScopesRows() throws InterruptedException { + final Map queryPart = + queryPartWithReference(SqlQueryExportTestConfiguration.PARAM_QUERY_ID); + final Map parametersResource = new LinkedHashMap<>(); + parametersResource.put("resourceType", "Parameters"); + parametersResource.put( + "parameter", List.of(Map.of("name", "familyName", "valueString", "Smith"))); + addPartTo(queryPart, resourcePart("parameters", parametersResource)); + + final Map manifest = + exportToCompletion(systemLevelUri(), parametersWith(queryPart)); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).contains("\"family_name\":\"Smith\""); + assertThat(content).doesNotContain("Johnson").doesNotContain("Williams"); + } + + // ------------------------------------------------------------------------- + // US6 - manifest shape, lifetime, and cancellation + // ------------------------------------------------------------------------- + + @Test + void manifestOmitsCancelUrlAndEstimatedTimeRemaining() throws InterruptedException { + final Map manifest = + exportToCompletion( + systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)); + assertThat(findParam(manifest, "cancelUrl")).isNull(); + assertThat(findParam(manifest, "estimatedTimeRemaining")).isNull(); + assertThat(findParam(manifest, "exportStartTime")).isNotNull(); + assertThat(findParam(manifest, "exportDuration")).isNotNull(); + } + + @Test + void resultUrlSupportsRepeatRetrievalWithExpiresHeader() throws InterruptedException { + final String resultLocation = + resultLocationOf( + systemLevelUri(), storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)); + + for (int i = 0; i < 2; i++) { + webTestClient + .get() + .uri(resultLocation) + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isOk() + .expectHeader() + .exists("Expires"); + } + } + + @Test + void deleteCancelsInProgressExportThenPollReturns404() { + final String contentLocation = + kickOff( + systemLevelUri(), + storedQuery(SqlQueryExportTestConfiguration.PATIENT_QUERY_ID, null)) + .expectStatus() + .isAccepted() + .returnResult(String.class) + .getResponseHeaders() + .getFirst("Content-Location"); + assertThat(contentLocation).isNotNull(); + + webTestClient.delete().uri(contentLocation).exchange().expectStatus().isAccepted(); + + webTestClient + .get() + .uri(contentLocation) + .header("Accept", "application/fhir+json") + .exchange() + .expectStatus() + .isNotFound(); + } + + // ------------------------------------------------------------------------- + // Request builders specific to this suite + // ------------------------------------------------------------------------- + + @Nonnull + private Map queryPartWithResource(@Nonnull final Map library) { + final Map queryParam = new LinkedHashMap<>(); + queryParam.put("name", "query"); + queryParam.put("part", new ArrayList<>(List.of(resourcePart("queryResource", library)))); + return queryParam; + } + + /** An inline SQLQuery Library referencing the stored Patient ViewDefinition by relative id. */ + @Nonnull + private Map inlineSqlLibrary() { + final Map library = new LinkedHashMap<>(); + library.put("resourceType", "Library"); + library.put("status", "active"); + library.put( + "type", + Map.of( + "coding", + List.of( + Map.of( + "system", + SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM, + "code", + SqlQueryLibraryParser.LIBRARY_TYPE_CODE)))); + final String sql = "SELECT id, family_name FROM patients ORDER BY id"; + library.put( + "content", + List.of( + Map.of( + "contentType", + "application/sql", + "data", + java.util.Base64.getEncoder() + .encodeToString(sql.getBytes(StandardCharsets.UTF_8))))); + library.put( + "relatedArtifact", + List.of( + Map.of( + "type", + "depends-on", + "label", + "patients", + "resource", + "ViewDefinition/" + SqlQueryExportTestConfiguration.PATIENT_VIEW_ID))); + return library; + } + + /** An inline Patient ViewDefinition whose id matches the inline query's relatedArtifact. */ + @Nonnull + private Map inlineViewDefinition() { + final Map view = new LinkedHashMap<>(); + view.put("resourceType", "ViewDefinition"); + view.put("id", SqlQueryExportTestConfiguration.PATIENT_VIEW_ID); + view.put("name", "supplied_patient_view"); + view.put("resource", "Patient"); + view.put("status", "active"); + view.put( + "select", + new ArrayList<>( + List.of( + Map.of( + "column", + List.of( + Map.of("name", "id", "path", "id"), + Map.of("name", "family_name", "path", "name.first().family")))))); + return view; + } + + @Nonnull + private Map viewPartWithResource(@Nonnull final Map view) { + final Map viewParam = new LinkedHashMap<>(); + viewParam.put("name", "view"); + viewParam.put("part", new ArrayList<>(List.of(resourcePart("viewResource", view)))); + return viewParam; + } + + @Nonnull + private Map viewPartWithReference(@Nonnull final String reference) { + final Map viewParam = new LinkedHashMap<>(); + viewParam.put("name", "view"); + viewParam.put("part", new ArrayList<>(List.of(referencePart("viewReference", reference)))); + return viewParam; + } + + @SuppressWarnings("unchecked") + private void addPartTo( + @Nonnull final Map param, @Nonnull final Map part) { + ((List>) param.get("part")).add(part); + } + + @Nonnull + private Map simplePart( + @Nonnull final String name, @Nonnull final String valueKey, @Nonnull final String value) { + final Map part = new LinkedHashMap<>(); + part.put("name", name); + part.put(valueKey, value); + return part; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java new file mode 100644 index 0000000000..3cbfea5acd --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java @@ -0,0 +1,184 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.view.ViewExecutionHelper; +import au.csiro.pathling.operations.view.ViewExportFormat; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import java.util.Map; +import java.util.Set; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Reference; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link SqlQueryExportRequestParser}, verifying source rejection, strict format + * parsing, query-source exclusivity, the no-query rejection, and the default format. The pipeline + * and resolver collaborators are mocked. + * + * @author John Grimes + */ +class SqlQueryExportRequestParserTest { + + private SqlQueryPipeline pipeline; + private LibraryReferenceResolver libraryReferenceResolver; + private SqlQueryExportRequestParser parser; + + @BeforeEach + void setUp() { + pipeline = mock(SqlQueryPipeline.class); + libraryReferenceResolver = mock(LibraryReferenceResolver.class); + final ViewExecutionHelper viewExecutionHelper = mock(ViewExecutionHelper.class); + final ServerConfiguration serverConfiguration = mock(ServerConfiguration.class); + final QueryableDataSource deltaLake = mock(QueryableDataSource.class); + parser = + new SqlQueryExportRequestParser( + pipeline, + libraryReferenceResolver, + viewExecutionHelper, + FhirContext.forR4Cached(), + serverConfiguration, + deltaLake); + + final ParsedSqlQuery parsedQuery = + new ParsedSqlQuery("SELECT id FROM patients", java.util.List.of(), java.util.List.of()); + final SqlQueryRequest request = + new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); + when(pipeline.prepare(any(), any(), any(), any(), any(), any(), any())) + .thenReturn(new PreparedSqlQuery(request, Map.of())); + when(libraryReferenceResolver.resolve(any())).thenReturn(new Library()); + } + + @Test + void parsesSingleStoredQueryWithDefaultNdjsonFormat() { + final Parameters body = parameters(queryWithReference("Library/patient-query")); + + final SqlQueryExportRequest request = + parser.parse(requestDetails(body), null, null, null, null, Set.of(), null, null); + + assertThat(request.queries()).hasSize(1); + assertThat(request.format()).isEqualTo(ViewExportFormat.NDJSON); + } + + @Test + void rejectsBothQuerySourcesWith400() { + final Parameters.ParametersParameterComponent query = + queryWithReference("Library/patient-query"); + query.addPart().setName("queryResource").setResource(new Library()); + final Parameters body = parameters(query); + + assertThatThrownBy( + () -> parser.parse(requestDetails(body), null, null, null, null, Set.of(), null, null)) + .isInstanceOf(InvalidRequestException.class); + } + + @Test + void rejectsNeitherQuerySourceWith400() { + final Parameters.ParametersParameterComponent query = new Parameters().addParameter(); + query.setName("query"); + query.addPart().setName("name").setValue(new org.hl7.fhir.r4.model.StringType("orphan")); + final Parameters body = parameters(query); + + assertThatThrownBy( + () -> parser.parse(requestDetails(body), null, null, null, null, Set.of(), null, null)) + .isInstanceOf(InvalidRequestException.class); + } + + @Test + void rejectsNoQueryAtSystemLevelWith400() { + assertThatThrownBy( + () -> + parser.parse( + requestDetails(new Parameters()), null, null, null, null, Set.of(), null, null)) + .isInstanceOf(InvalidRequestException.class); + } + + @Test + void rejectsSourceParameterWith400BeforeResolvingAnyQuery() { + final Parameters body = parameters(queryWithReference("Library/patient-query")); + + assertThatThrownBy( + () -> + parser.parse( + requestDetails(body), null, null, null, null, Set.of(), null, "s3://bucket")) + .isInstanceOf(InvalidRequestException.class); + verifyNoInteractions(pipeline, libraryReferenceResolver); + } + + @Test + void rejectsUnsupportedFormatWith400() { + final Parameters body = parameters(queryWithReference("Library/patient-query")); + + assertThatThrownBy( + () -> + parser.parse(requestDetails(body), null, "json", null, null, Set.of(), null, null)) + .isInstanceOf(InvalidRequestException.class); + } + + @Test + void missingQueryReferenceLibraryPropagatesNotFound() { + when(libraryReferenceResolver.resolve(any())) + .thenThrow(new ResourceNotFoundException("Library with ID 'missing' not found")); + final Parameters body = parameters(queryWithReference("Library/missing")); + + assertThatThrownBy( + () -> parser.parse(requestDetails(body), null, null, null, null, Set.of(), null, null)) + .isInstanceOf(ResourceNotFoundException.class); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private static Parameters.ParametersParameterComponent queryWithReference(final String ref) { + final Parameters.ParametersParameterComponent query = + new Parameters.ParametersParameterComponent().setName("query"); + query.addPart().setName("queryReference").setValue(new Reference(ref)); + return query; + } + + private static Parameters parameters(final Parameters.ParametersParameterComponent... params) { + final Parameters parameters = new Parameters(); + for (final Parameters.ParametersParameterComponent param : params) { + parameters.addParameter(param); + } + return parameters; + } + + private static ServletRequestDetails requestDetails(final Parameters body) { + final ServletRequestDetails requestDetails = mock(ServletRequestDetails.class); + when(requestDetails.getResource()).thenReturn(body); + when(requestDetails.getCompleteUrl()).thenReturn("http://localhost/fhir/$sqlquery-export"); + when(requestDetails.getFhirServerBase()).thenReturn("http://localhost/fhir"); + return requestDetails; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java new file mode 100644 index 0000000000..06dd6c1c61 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java @@ -0,0 +1,242 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_CODE; +import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM; + +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; +import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; +import au.csiro.pathling.library.PathlingContext; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.util.CustomObjectDataSource; +import jakarta.annotation.Nonnull; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.spark.sql.SparkSession; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Attachment; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Observation.ObservationStatus; +import org.hl7.fhir.r4.model.ParameterDefinition; +import org.hl7.fhir.r4.model.ParameterDefinition.ParameterUse; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.RelatedArtifact; +import org.hl7.fhir.r4.model.RelatedArtifact.RelatedArtifactType; +import org.hl7.fhir.r4.model.StringType; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; + +/** + * Test configuration that overrides the production {@code deltaLake} data source with an in-memory + * one pre-loaded with FHIR data, ViewDefinitions, and stored SQLQuery {@code Library} resources, + * for the {@code $sqlquery-export} integration tests. The stored Libraries reference stored + * ViewDefinitions via {@code relatedArtifact}, exercising the full storage-backed resolution path + * the production warehouse caching otherwise defeats for runtime-created resources. + * + * @author John Grimes + */ +@TestConfiguration +public class SqlQueryExportTestConfiguration { + + /** Id of the stored Patient ViewDefinition referenced by the patient query. */ + public static final String PATIENT_VIEW_ID = "patient-bp"; + + /** Id of the stored Observation ViewDefinition referenced by the observation query. */ + public static final String OBSERVATION_VIEW_ID = "observation-weight"; + + /** Id of the stored SQLQuery Library that selects patients. */ + public static final String PATIENT_QUERY_ID = "patient-bp-query"; + + /** Id of the stored SQLQuery Library that selects observations. */ + public static final String OBSERVATION_QUERY_ID = "observation-weight-query"; + + /** Id of the stored SQLQuery Library that declares a {@code familyName} parameter. */ + public static final String PARAM_QUERY_ID = "patient-by-family-query"; + + /** Id of the stored Group whose sole member is patient {@code p1}. */ + public static final String GROUP_ID = "g1"; + + /** The {@code meta.lastUpdated} instant of patient {@code p1} (older than the others). */ + public static final String P1_LAST_UPDATED = "2020-01-01T00:00:00.000Z"; + + @Primary + @Bean + @Nonnull + public QueryableDataSource deltaLake( + @Nonnull final SparkSession sparkSession, + @Nonnull final PathlingContext pathlingContext, + @Nonnull final FhirEncoders fhirEncoders) { + final List resources = new ArrayList<>(); + resources.add(patientView()); + resources.add(observationView()); + resources.add( + sqlLibrary( + PATIENT_QUERY_ID, + "patient_bp_query", + "SELECT id, family_name FROM patients ORDER BY id", + "patients", + "ViewDefinition/" + PATIENT_VIEW_ID)); + resources.add( + sqlLibrary( + OBSERVATION_QUERY_ID, + "observation_weight_query", + "SELECT id, subject, weight_kg FROM observations ORDER BY id", + "observations", + "ViewDefinition/" + OBSERVATION_VIEW_ID)); + resources.add(parameterisedPatientQuery()); + // p1 has an older meta.lastUpdated than p2/p3, so a `_since` filter can scope it out. + resources.add(patient("p1", "Smith", P1_LAST_UPDATED)); + resources.add(patient("p2", "Johnson", "2025-01-01T00:00:00.000Z")); + resources.add(patient("p3", "Williams", "2025-01-01T00:00:00.000Z")); + resources.add(observation("o1", "p1", new BigDecimal("70.50"))); + resources.add(observation("o2", "p2", new BigDecimal("82.250"))); + resources.add(observation("o3", "p3", new BigDecimal("65"))); + // A Group whose sole member is p1, for the group-filter test. + resources.add(group(GROUP_ID, "p1")); + return new CustomObjectDataSource(sparkSession, pathlingContext, fhirEncoders, resources); + } + + @Nonnull + private static ViewDefinitionResource patientView() { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(PATIENT_VIEW_ID); + view.setName(new StringType("patient_view")); + view.setResource(new CodeType("Patient")); + view.setStatus(new CodeType("active")); + final SelectComponent select = new SelectComponent(); + select.getColumn().add(column("id", "id")); + select.getColumn().add(column("family_name", "name.first().family")); + view.getSelect().add(select); + return view; + } + + @Nonnull + private static ViewDefinitionResource observationView() { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(OBSERVATION_VIEW_ID); + view.setName(new StringType("observation_view")); + view.setResource(new CodeType("Observation")); + view.setStatus(new CodeType("active")); + final SelectComponent select = new SelectComponent(); + select.getColumn().add(column("id", "id")); + select.getColumn().add(column("subject", "subject.reference")); + select.getColumn().add(column("weight_kg", "value.ofType(Quantity).value.toString()")); + view.getSelect().add(select); + return view; + } + + /** A stored SQLQuery Library declaring a {@code familyName} string parameter. */ + @Nonnull + private static Library parameterisedPatientQuery() { + final Library library = + sqlLibrary( + PARAM_QUERY_ID, + "patient_by_family_query", + "SELECT id, family_name FROM patients WHERE family_name = :familyName", + "patients", + "ViewDefinition/" + PATIENT_VIEW_ID); + library.addParameter( + new ParameterDefinition().setName("familyName").setUse(ParameterUse.IN).setType("string")); + return library; + } + + /** Builds a stored SQLQuery Library referencing a single ViewDefinition. */ + @Nonnull + static Library sqlLibrary( + @Nonnull final String id, + @Nonnull final String name, + @Nonnull final String sql, + @Nonnull final String label, + @Nonnull final String viewReference) { + final Library library = new Library(); + library.setId(id); + library.setName(name); + library.setStatus(PublicationStatus.ACTIVE); + library.setType( + new CodeableConcept() + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(LIBRARY_TYPE_CODE))); + final Attachment content = new Attachment(); + content.setContentType("application/sql"); + content.setData(sql.getBytes(StandardCharsets.UTF_8)); + library.addContent(content); + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel(label) + .setResource(viewReference)); + return library; + } + + @Nonnull + private static ColumnComponent column(@Nonnull final String name, @Nonnull final String path) { + final ColumnComponent column = new ColumnComponent(); + column.setName(new StringType(name)); + column.setPath(new StringType(path)); + return column; + } + + @Nonnull + private static Patient patient( + @Nonnull final String id, @Nonnull final String family, @Nonnull final String lastUpdated) { + final Patient patient = new Patient(); + patient.setId(id); + patient.addName().setFamily(family); + patient.getMeta().setLastUpdatedElement(new org.hl7.fhir.r4.model.InstantType(lastUpdated)); + return patient; + } + + @Nonnull + private static org.hl7.fhir.r4.model.Group group( + @Nonnull final String id, @Nonnull final String memberPatientId) { + final org.hl7.fhir.r4.model.Group fhirGroup = new org.hl7.fhir.r4.model.Group(); + fhirGroup.setId(id); + fhirGroup.setType(org.hl7.fhir.r4.model.Group.GroupType.PERSON); + fhirGroup.setActual(true); + fhirGroup.addMember().setEntity(new Reference("Patient/" + memberPatientId)); + return fhirGroup; + } + + @Nonnull + private static Observation observation( + @Nonnull final String id, @Nonnull final String patientId, @Nonnull final BigDecimal weight) { + final Observation observation = new Observation(); + observation.setId(id); + observation.setStatus(ObservationStatus.FINAL); + observation.setSubject(new Reference("Patient/" + patientId)); + final Quantity quantity = new Quantity(); + quantity.setValue(weight); + quantity.setUnit("kg"); + quantity.setSystem("http://unitsofmeasure.org"); + quantity.setCode("kg"); + observation.setValue(quantity); + return observation; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java new file mode 100644 index 0000000000..320f95dcb2 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.io.source.DataSource; +import au.csiro.pathling.views.FhirView; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Library; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link SqlQueryPipeline}, verifying that it orchestrates parsing, view resolution, + * static validation, and execution across its collaborators. The collaborators are mocked, so the + * test exercises the pipeline's wiring rather than the Spark-backed execution itself. + * + * @author John Grimes + */ +class SqlQueryPipelineTest { + + private SqlQueryRequestParser requestParser; + private ViewResolver viewResolver; + private SqlQueryExecutor executor; + private SqlValidator sqlValidator; + private SqlQueryPipeline pipeline; + + private Library library; + private SqlQueryRequest request; + private Map resolvedViews; + + @BeforeEach + void setUp() { + requestParser = mock(SqlQueryRequestParser.class); + viewResolver = mock(ViewResolver.class); + executor = mock(SqlQueryExecutor.class); + sqlValidator = mock(SqlValidator.class); + pipeline = new SqlQueryPipeline(requestParser, viewResolver, executor, sqlValidator); + + library = new Library(); + final ParsedSqlQuery parsedQuery = + new ParsedSqlQuery( + "SELECT id FROM patients", + List.of(new ViewArtifactReference("patients", "ViewDefinition/patient-view")), + List.of()); + request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); + final FhirView view = mock(FhirView.class); + resolvedViews = Map.of("patients", view); + } + + @Test + void prepareParsesAndResolvesViewsWithSuppliedViews() { + final FhirView suppliedView = mock(FhirView.class); + final Map supplied = Map.of("patient-view", suppliedView); + when(requestParser.parse(eq(library), eq("ndjson"), any(), any(), any(), any())) + .thenReturn(request); + when(viewResolver.resolve(request.getParsedQuery().getViewReferences(), supplied)) + .thenReturn(resolvedViews); + + final PreparedSqlQuery prepared = + pipeline.prepare(library, "ndjson", null, null, null, null, supplied); + + assertThat(prepared.getRequest()).isSameAs(request); + assertThat(prepared.getResolvedViews()).isEqualTo(resolvedViews); + verify(requestParser).parse(eq(library), eq("ndjson"), any(), any(), any(), any()); + verify(viewResolver).resolve(request.getParsedQuery().getViewReferences(), supplied); + } + + @Test + void validateStaticallyValidatesSqlWithDeclaredLabels() { + final PreparedSqlQuery prepared = new PreparedSqlQuery(request, resolvedViews); + + pipeline.validateStatically(prepared); + + verify(sqlValidator).validate("SELECT id FROM patients", Set.of("patients")); + } + + @Test + void executeDelegatesToExecutorAndPassesDatasetToConsumer() { + final PreparedSqlQuery prepared = new PreparedSqlQuery(request, resolvedViews); + final DataSource dataSource = mock(DataSource.class); + @SuppressWarnings("unchecked") + final Dataset dataset = mock(Dataset.class); + + // Have the mocked executor invoke the terminal consumer with the dataset, as the real one does. + doAnswer( + invocation -> { + final Consumer> consumer = invocation.getArgument(4); + consumer.accept(dataset); + return null; + }) + .when(executor) + .execute(eq(request), eq(resolvedViews), eq(dataSource), eq("req-1"), any()); + + final AtomicReference> received = new AtomicReference<>(); + pipeline.execute(prepared, dataSource, "req-1", received::set); + + assertThat(received.get()).isSameAs(dataset); + verify(executor).execute(eq(request), eq(resolvedViews), eq(dataSource), eq("req-1"), any()); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutorTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutorTest.java index 09578cad91..767c238e3a 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutorTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportExecutorTest.java @@ -25,6 +25,8 @@ import au.csiro.pathling.library.PathlingContext; import au.csiro.pathling.library.io.source.QueryableDataSource; import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import au.csiro.pathling.operations.export.ExportDataSourceBuilder; +import au.csiro.pathling.operations.export.ExportFileWriter; import au.csiro.pathling.test.SharedMocks; import au.csiro.pathling.test.SpringBootUnitTest; import au.csiro.pathling.util.CustomObjectDataSource; @@ -338,6 +340,39 @@ void invalidViewDefinitionThrowsException() { .isInstanceOf(InvalidRequestException.class); } + // ------------------------------------------------------------------------- + // Patient compartment filtering (shared ExportDataSourceBuilder) + // ------------------------------------------------------------------------- + + @Test + void patientCompartmentFilterScopesAndHandlesFhirPathPaths() { + // The data source mixes Patient (filtered by id) with Observation, whose compartment membership + // is defined by a FHIRPath expression (subject.where(resolve() is Patient)). The shared builder + // must filter both correctly without failing on the FHIRPath path. + final Patient p1 = createPatient("p1", "Smith"); + final Patient p2 = createPatient("p2", "Jones"); + final var obs = createObservation("o1", "p1"); + executor = createExecutor(p1, p2, obs); + + final FhirView view = createSimplePatientView(); + final ViewInput viewInput = new ViewInput("patients", view); + final ViewDefinitionExportRequest request = + new ViewDefinitionExportRequest( + "http://example.org/$viewdefinition-export", + "http://example.org/fhir", + List.of(viewInput), + null, + ViewExportFormat.NDJSON, + true, + java.util.Set.of("p1"), + null); + + // Must not throw on the Observation FHIRPath compartment path, and must produce one output. + final List outputs = executor.execute(request, UUID.randomUUID().toString()); + assertThat(outputs).hasSize(1); + assertThat(outputs.get(0).fileUrls()).isNotEmpty(); + } + // ------------------------------------------------------------------------- // Helper methods // ------------------------------------------------------------------------- @@ -345,13 +380,12 @@ void invalidViewDefinitionThrowsException() { private ViewDefinitionExportExecutor createExecutor(final IBaseResource... resources) { final QueryableDataSource dataSource = new CustomObjectDataSource(sparkSession, pathlingContext, fhirEncoders, List.of(resources)); + final ExportFileWriter fileWriter = + new ExportFileWriter(sparkSession, "file://" + uniqueTempDir.toAbsolutePath()); + final ExportDataSourceBuilder dataSourceBuilder = + new ExportDataSourceBuilder(patientCompartmentService); return new ViewDefinitionExportExecutor( - dataSource, - fhirContext, - sparkSession, - "file://" + uniqueTempDir.toAbsolutePath(), - serverConfiguration, - patientCompartmentService); + dataSource, fhirContext, serverConfiguration, dataSourceBuilder, fileWriter); } private Patient createPatient(final String id, final String familyName) { @@ -361,6 +395,15 @@ private Patient createPatient(final String id, final String familyName) { return patient; } + private org.hl7.fhir.r4.model.Observation createObservation( + final String id, final String patientId) { + final org.hl7.fhir.r4.model.Observation observation = new org.hl7.fhir.r4.model.Observation(); + observation.setId(id); + observation.setStatus(org.hl7.fhir.r4.model.Observation.ObservationStatus.FINAL); + observation.setSubject(new org.hl7.fhir.r4.model.Reference("Patient/" + patientId)); + return observation; + } + private FhirView createSimplePatientView() { return FhirView.ofResource("Patient") .select( diff --git a/site/docs/server/configuration.md b/site/docs/server/configuration.md index 28eed9a6a0..393ef97110 100644 --- a/site/docs/server/configuration.md +++ b/site/docs/server/configuration.md @@ -163,6 +163,9 @@ error response and is excluded from the CapabilityStatement. ViewDefinition resources. - `pathling.operations.viewDefinitionExportEnabled` - (default: `true`) Enables the [$viewdefinition-export](./operations/view-export) operation. +- `pathling.operations.sqlQueryExportEnabled` - (default: `true`) Enables the + system, type, and instance-level + [$sqlquery-export](./operations/sql-export) operation. - `pathling.operations.bulkSubmitEnabled` - (default: `true`) Enables the [$bulk-submit](./operations/bulk-submit) operation. diff --git a/site/docs/server/operations/sql-export.md b/site/docs/server/operations/sql-export.md new file mode 100644 index 0000000000..a5a84a836f --- /dev/null +++ b/site/docs/server/operations/sql-export.md @@ -0,0 +1,231 @@ +--- +sidebar_position: 10 +description: The sqlquery-export operation asynchronously runs one or more SQL queries against materialised ViewDefinition tables and exports the results to downloadable files in NDJSON, CSV, or Parquet format. +--- + +# Export SQL query + +This operation is the asynchronous counterpart to the +[run SQL query](sql-run.md) operation. It runs one or more SQL queries against +materialised +[ViewDefinition](https://build.fhir.org/ig/FHIR/sql-on-fhir-v2/StructureDefinition-ViewDefinition.html) +tables and exports the results to downloadable files, following the +[SQL on FHIR specification](http://sql-on-fhir.org/OperationDefinition/$sqlquery-export) +and the [FHIR Asynchronous Request Pattern](https://hl7.org/fhir/R4/async.html). + +Use this operation for result sets that are too large or too slow to retrieve +synchronously with [run SQL query](sql-run.md). Each query produces one +downloadable output. + +## Endpoints + +The operation is invocable at the system, type, and instance levels: + +``` +POST [base]/$sqlquery-export +POST [base]/Library/$sqlquery-export +POST [base]/Library/[id]/$sqlquery-export +``` + +At the instance level the bound `Library` is the single query source; the +`query` parameter does not apply, and per-query parameter binding is not offered +at that level. + +All invocations require the `Prefer: respond-async` header. + +## Parameters + +| Name | Cardinality | Type | Description | +| ---------------------- | ----------- | ---------- | ---------------------------------------------------------------------------------------------------------- | +| `query` | 1..\* | (parts) | A repeating parameter (system and type levels only); each repetition is one query and produces one output. | +| `query.name` | 0..1 | string | Optional output name. Highest precedence in the output-name derivation. | +| `query.queryReference` | 0..1 | Reference | A reference to a stored SQLQuery `Library`. Mutually exclusive with `query.queryResource`. | +| `query.queryResource` | 0..1 | Resource | An inline SQLQuery `Library`. Mutually exclusive with `query.queryReference`. | +| `query.parameters` | 0..1 | Parameters | Per-query runtime parameter bindings, bound by name to the Library's declared parameters. | +| `view` | 0..\* | (parts) | A repeating parameter supplying ViewDefinition table sources at request time. | +| `view.name` | 0..1 | string | Optional friendly label. Not used for matching or as an output name. | +| `view.viewReference` | 0..1 | Reference | A reference to a stored ViewDefinition. Mutually exclusive with `view.viewResource`. | +| `view.viewResource` | 0..1 | Resource | An inline ViewDefinition. Mutually exclusive with `view.viewReference`. | +| `clientTrackingId` | 0..1 | string | Client-provided tracking identifier; echoed in the acknowledgement and the completion manifest. | +| `_format` | 0..1 | code | Output format: `ndjson` (default), `csv`, or `parquet`. An unsupported value returns `400`. | +| `header` | 0..1 | boolean | Include a header row in CSV output. Defaults to `true`. Has no effect on NDJSON or Parquet. | +| `patient` | 0..\* | Reference | Filter to resources for the specified patient(s). | +| `group` | 0..\* | Reference | Filter to resources for patients in the specified Group(s). | +| `_since` | 0..1 | instant | Only include resources where `meta.lastUpdated` is at or after this time. | + +### Query and table sources + +Each `query` part must supply exactly one of `query.queryReference` or +`query.queryResource`; supplying both, or neither, returns `400 Bad Request`. A +`query.queryReference` that does not resolve to a stored Library returns +`404 Not Found`. + +A SQLQuery `Library` declares its table sources as `relatedArtifact` entries, +each labelling a ViewDefinition the SQL references. The optional `view` +parameter supplies those ViewDefinitions at request time, matched to the +`relatedArtifact` entries by ViewDefinition id. A view the SQL references but no +`view` part supplies is read from server storage, exactly as the synchronous +operation does. Each `view` part must supply exactly one of `view.viewReference` +or `view.viewResource`; supplying both, or neither, returns `400 Bad Request`. A +supplied ViewDefinition that is well-formed but semantically invalid returns +`422 Unprocessable Entity`. + +The `source` parameter (an external data source) is not supported by this +server; supplying it returns `400 Bad Request`. All of these rejections are +returned synchronously at kick-off. + +## Asynchronous processing + +```mermaid +sequenceDiagram + participant C as Client + participant P as Pathling + C->>P: POST /$sqlquery-export (Prefer: respond-async) + P-->>C: 202 Accepted + Content-Location + loop Poll for status + C->>P: GET [status URL] + P-->>C: 202 Accepted (in progress) + end + C->>P: GET [status URL] + P-->>C: 303 See Other + Location + C->>P: GET [result URL] + P-->>C: 200 OK + manifest + C->>P: GET $result?job=...&file=... + P-->>C: Exported data file +``` + +### Kick-off request + +```http +POST [base]/Library/$sqlquery-export HTTP/1.1 +Content-Type: application/fhir+json +Accept: application/fhir+json +Prefer: respond-async + +{ + "resourceType": "Parameters", + "parameter": [ + { + "name": "query", + "part": [ + {"name": "name", "valueString": "people"}, + { + "name": "queryReference", + "valueReference": {"reference": "Library/patient-bp-query"} + } + ] + } + ] +} +``` + +### Kick-off response + +The `202 Accepted` response carries a `Parameters` acknowledgement and a +`Content-Location` header pointing at the status URL: + +```http +HTTP/1.1 202 Accepted +Content-Location: [base]/$job?id=[job-id] + +{ + "resourceType": "Parameters", + "parameter": [ + {"name": "status", "valueCode": "accepted"}, + {"name": "exportId", "valueString": "[job-id]"} + ] +} +``` + +### Polling + +Poll the URL from `Content-Location`: + +- `202 Accepted` — Export still in progress. Check the `X-Progress` header. +- `303 See Other` — Export complete; follow the `Location` header to the result + URL, then `GET` it for the manifest. +- `404 Not Found` — The export was cancelled (via `DELETE` on the status URL) or + is unknown. + +## Response manifest + +The completion manifest is a FHIR `Parameters` resource following the SQL on +FHIR shape: + +```json +{ + "resourceType": "Parameters", + "parameter": [ + { "name": "exportId", "valueString": "abc123" }, + { "name": "status", "valueCode": "completed" }, + { "name": "clientTrackingId", "valueString": "my-tracking-id" }, + { "name": "_format", "valueCode": "ndjson" }, + { + "name": "exportStartTime", + "valueInstant": "2026-06-21T01:00:00.000Z" + }, + { "name": "exportEndTime", "valueInstant": "2026-06-21T01:00:12.000Z" }, + { "name": "exportDuration", "valueInteger": 12 }, + { + "name": "output", + "part": [ + { "name": "name", "valueString": "people" }, + { + "name": "location", + "valueUri": "https://pathling.example.com/fhir/$result?job=abc123&file=people.00000.ndjson" + } + ] + } + ] +} +``` + +There is one `output` per query, each with a `name` part and one or more +`location` download URLs (a query partitioned into several files repeats the +`location` part once per file). The manifest does not include the optional +`cancelUrl` or `estimatedTimeRemaining` parameters. + +## Output formats + +| Format | `_format` value | Content type | Description | +| ------- | --------------- | -------------------------------- | -------------------------------------------------------------- | +| NDJSON | `ndjson` | `application/x-ndjson` | Newline-delimited JSON. Default format. | +| CSV | `csv` | `text/csv` | Comma-separated values. Use `header=false` to exclude headers. | +| Parquet | `parquet` | `application/vnd.apache.parquet` | Apache Parquet columnar format. Efficient for large datasets. | + +## Multiple queries + +Include several `query` parameters to export several result sets in one +operation. Each produces one named output. The `output.name` is derived as the +`query.name` when supplied, otherwise the Library's `name` element, otherwise a +generated unique name. If any query fails, the whole export fails (all or +nothing): no manifest is produced and the result URL returns the error status +with an OperationOutcome. + +## Filtering + +The `patient`, `group`, and `_since` parameters scope the exported rows in the +same way as the [export view](view-export.md) operation. + +## Cancellation and lifetime + +Send a `DELETE` to the status URL to cancel an in-progress export; subsequent +polls of that URL return `404 Not Found`. The result and download URLs remain +valid for at least 24 hours after completion and support repeat retrieval. + +## Comparison with run SQL query + +| Aspect | Export SQL query | Run SQL query | +| ------------------ | ---------------------------------- | ------------------------------------------ | +| Processing | Asynchronous with polling | Synchronous | +| Output | Files (download via manifest URLs) | Streamed response | +| Multiple queries | Yes | No | +| Request-time views | Yes (via the `view` parameter) | No | +| Output formats | `ndjson`, `csv`, `parquet` | `ndjson`, `csv`, `json`, `parquet`, `fhir` | +| Use case | Large result sets, batch export | Small queries, interactive use | + +## Configuration + +The operation is enabled by default and can be disabled with the +`pathling.operations.sqlQueryExportEnabled` configuration setting (see +[Configuration](../configuration.md)). diff --git a/ui/CONTRIBUTING.md b/ui/CONTRIBUTING.md index d3175deb6d..29f9c52045 100644 --- a/ui/CONTRIBUTING.md +++ b/ui/CONTRIBUTING.md @@ -129,6 +129,13 @@ bun run test:e2e:headed rendering or to handle user events. Effects are escape hatches, not primary tools. +The asynchronous export operations share a single presentational card, +`components/sqlOnFhir/ExportJobCard.tsx`. It is parameterised by the export +format and a manifest-output parser, and is reused by both the ViewDefinition +export (`ViewExportCard`) and the SQL query export (`SqlQueryExportCardWrapper`) +flows. When adding a new export-style operation, reuse `ExportJobCard` rather +than duplicating the status, progress, cancel, and download presentation. + ### Naming | Element | Convention | Example | diff --git a/ui/e2e/sqlQueryExport.spec.ts b/ui/e2e/sqlQueryExport.spec.ts new file mode 100644 index 0000000000..720d2c322e --- /dev/null +++ b/ui/e2e/sqlQueryExport.spec.ts @@ -0,0 +1,194 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * E2E tests for the `$sqlquery-export` affordance on the SQL on FHIR page: running a query, then + * exporting its result set asynchronously and downloading an output file. All network is mocked + * with `page.route`. + * + * @author John Grimes + */ + +import { expect, test } from "@playwright/test"; + +import { + mockCapabilityStatement, + mockSqlQueryLibrary1, + mockSqlQueryLibraryBundle, + mockSqlQueryRunCsv, + mockViewDefinitionBundle, +} from "./fixtures/fhirData"; + +import type { Page } from "@playwright/test"; + +/** + * Mocks the base endpoints needed to load the page and run a stored SQL query. + * + * @param page - The Playwright page. + */ +async function mockBaseEndpoints(page: Page) { + await page.route("**/metadata", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/fhir+json", + body: JSON.stringify(mockCapabilityStatement), + }); + }); + + await page.route(/\/Library\?[^"]*$/, async (route) => { + const url = route.request().url(); + const body = url.includes("sql-query") + ? mockSqlQueryLibraryBundle + : { resourceType: "Bundle", type: "searchset", total: 0, entry: [] }; + await route.fulfill({ + status: 200, + contentType: "application/fhir+json", + body: JSON.stringify(body), + }); + }); + + const viewDefinitions = (route: import("@playwright/test").Route) => + route.fulfill({ + status: 200, + contentType: "application/fhir+json", + body: JSON.stringify(mockViewDefinitionBundle), + }); + await page.route("**/ViewDefinition?*", viewDefinitions); + await page.route(/\/ViewDefinition$/, viewDefinitions); + + await page.route("**/$sqlquery-run", async (route) => { + await route.fulfill({ + status: 200, + contentType: "text/csv", + body: mockSqlQueryRunCsv, + }); + }); +} + +/** + * Mocks the asynchronous export endpoints: kick-off, status poll (returning the completion + * manifest), and the output-file download. + * + * @param page - The Playwright page. + */ +async function mockExportEndpoints(page: Page) { + // Kick-off returns 202 with a polling URL. + await page.route(/\/\$sqlquery-export/, async (route) => { + await route.fulfill({ + status: 202, + headers: { + "Content-Location": "http://localhost:3000/fhir/$job?id=sql-export-1", + "Access-Control-Expose-Headers": "Content-Location", + }, + body: "", + }); + }); + + // Status poll returns the completion manifest, with one output file. + await page.route(/\/\$job/, async (route) => { + if (route.request().method() === "DELETE") { + await route.fulfill({ status: 204 }); + return; + } + await route.fulfill({ + status: 200, + contentType: "application/fhir+json", + body: JSON.stringify({ + resourceType: "Parameters", + parameter: [ + { name: "exportId", valueString: "sql-export-1" }, + { name: "status", valueCode: "completed" }, + { name: "_format", valueCode: "ndjson" }, + { + name: "output", + part: [ + { name: "name", valueString: "people" }, + { + name: "location", + valueUri: + "http://localhost:3000/fhir/$result?job=sql-export-1&file=people.00000.ndjson", + }, + ], + }, + ], + }), + }); + }); + + // The output-file download. + await page.route(/\/\$result/, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/octet-stream", + body: '{"id":"p1","family_name":"Smith"}\n', + }); + }); +} + +/** + * Runs the stored SQL query so the result card renders with rows. + * + * @param page - The Playwright page. + */ +async function runStoredQuery(page: Page) { + await page.goto("/admin/sql-on-fhir"); + await page.getByRole("tab", { name: /^sql query$/i }).click(); + + await page.getByRole("combobox", { name: /sql query library/i }).click(); + await page.getByRole("option", { name: mockSqlQueryLibrary1.title }).click(); + + // Enter a runtime value for the declared parameter. + await page + .getByRole("textbox", { name: /runtime value for patient_id/i }) + .fill("Patient/pat-1"); + + // Use CSV so the response branch is deterministic and returns rows. + await page.getByRole("combobox", { name: /output format/i }).click(); + await page.getByRole("option", { name: "csv" }).click(); + + await page.getByRole("button", { name: /^execute$/i }).click(); +} + +test.describe("SQL on FHIR page - SQL query export", () => { + test("exports a query result and lists the downloadable output", async ({ + page, + }) => { + await mockBaseEndpoints(page); + await mockExportEndpoints(page); + + await runStoredQuery(page); + + // The run completes and renders a result table. + await expect(page.getByText("2 rows")).toBeVisible(); + + // The result card offers the export affordance. + await expect(page.getByText(/export full result set/i)).toBeVisible(); + + // Start the export. + await page.getByRole("button", { name: /^export$/i }).click(); + + // The export job card completes and lists the output file. + await expect(page.getByText(/completed/i)).toBeVisible(); + await expect(page.getByText("people.00000.ndjson")).toBeVisible(); + + // Downloading the output triggers a browser download. + const downloadPromise = page.waitForEvent("download"); + await page.getByRole("button", { name: /^download$/i }).click(); + const download = await downloadPromise; + expect(download.suggestedFilename()).toContain("people.00000.ndjson"); + }); +}); diff --git a/ui/src/api/__tests__/sqlQuery.test.ts b/ui/src/api/__tests__/sqlQuery.test.ts index 4ec38ed92a..3cd2d097fd 100644 --- a/ui/src/api/__tests__/sqlQuery.test.ts +++ b/ui/src/api/__tests__/sqlQuery.test.ts @@ -17,10 +17,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { OperationOutcomeError, UnauthorizedError } from "../../types/errors"; +import { + NotFoundError, + OperationOutcomeError, + UnauthorizedError, +} from "../../types/errors"; import { listSqlQueryLibraries, SQL_QUERY_LIBRARY_TYPE_FILTER, + sqlQueryExportDownload, + sqlQueryExportKickOff, sqlQueryRun, } from "../sqlQuery"; @@ -366,3 +372,81 @@ describe("listSqlQueryLibraries", () => { ).rejects.toThrow(UnauthorizedError); }); }); + +describe("sqlQueryExportKickOff", () => { + // The kick-off must request asynchronous processing and return the polling + // URL carried by the Content-Location header. + it("sets Prefer: respond-async and returns the polling URL", async () => { + mockFetch.mockResolvedValueOnce( + new Response(null, { + status: 202, + headers: { "Content-Location": "https://example.com/fhir/$job?id=abc" }, + }), + ); + + const result = await sqlQueryExportKickOff("https://example.com/fhir", { + request: { + mode: "stored", + libraryId: "patient-bp-query", + format: "ndjson", + }, + }); + + expect(result.pollingUrl).toBe("https://example.com/fhir/$job?id=abc"); + expect(mockFetch).toHaveBeenCalledWith( + "https://example.com/fhir/$sqlquery-export", + expect.objectContaining({ + method: "POST", + headers: expect.objectContaining({ Prefer: "respond-async" }), + }), + ); + }); + + // Without a Content-Location header the kick-off cannot be polled, so it + // fails rather than returning an undefined URL. + it("throws when no Content-Location header is present", async () => { + mockFetch.mockResolvedValueOnce(new Response(null, { status: 202 })); + await expect( + sqlQueryExportKickOff("https://example.com/fhir", { + request: { mode: "stored", libraryId: "q", format: "ndjson" }, + }), + ).rejects.toThrow("No Content-Location"); + }); +}); + +describe("sqlQueryExportDownload", () => { + it("returns the response body stream on success", async () => { + const body = new ReadableStream(); + mockFetch.mockResolvedValueOnce(new Response(body, { status: 200 })); + + const stream = await sqlQueryExportDownload({ + location: "https://example.com/fhir/$result?job=j&file=people.ndjson", + }); + + expect(stream).toBe(body); + expect(mockFetch).toHaveBeenCalledWith( + "https://example.com/fhir/$result?job=j&file=people.ndjson", + expect.objectContaining({ method: "GET" }), + ); + }); + + it("throws NotFoundError on a 404 response", async () => { + mockFetch.mockResolvedValueOnce(new Response("Not found", { status: 404 })); + await expect( + sqlQueryExportDownload({ + location: "https://example.com/fhir/$result?job=j&file=x", + }), + ).rejects.toThrow(NotFoundError); + }); + + it("throws UnauthorizedError on a 401 response", async () => { + mockFetch.mockResolvedValueOnce( + new Response("Unauthorized", { status: 401 }), + ); + await expect( + sqlQueryExportDownload({ + location: "https://example.com/fhir/$result?job=j&file=x", + }), + ).rejects.toThrow(UnauthorizedError); + }); +}); diff --git a/ui/src/api/index.ts b/ui/src/api/index.ts index 4093141b1b..5bac26cc9e 100644 --- a/ui/src/api/index.ts +++ b/ui/src/api/index.ts @@ -64,6 +64,8 @@ export type { ViewDefinition } from "./view"; export { sqlQueryRun, listSqlQueryLibraries, + sqlQueryExportKickOff, + sqlQueryExportDownload, SQL_QUERY_LIBRARY_TYPE_SYSTEM, SQL_QUERY_LIBRARY_TYPE_FILTER, SQL_QUERY_LIBRARY_PROFILE, @@ -72,4 +74,7 @@ export type { SqlQueryRunOptions, SqlQueryRunStoredOptions, SqlQueryRunInlineOptions, + SqlQueryExportKickOffOptions, + SqlQueryExportResult, + SqlQueryExportDownloadOptions, } from "./sqlQuery"; diff --git a/ui/src/api/sqlQuery.ts b/ui/src/api/sqlQuery.ts index 838bc43a8a..e8fc0cf310 100644 --- a/ui/src/api/sqlQuery.ts +++ b/ui/src/api/sqlQuery.ts @@ -28,9 +28,11 @@ import { checkResponse, pushOutputParameters, } from "./utils"; +import { buildSqlQueryExportKickOffBody } from "../hooks/sqlQueryExportHelpers"; import type { AuthOptions } from "./rest"; import type { + SqlQueryExportRequest, SqlQueryLibrary, SqlQueryOutputFormat, SqlQueryParameterType, @@ -302,3 +304,114 @@ function bindingToPart( return { name, valueDateTime: rawValue }; } } + +// ============================================================================= +// SQL query export +// ============================================================================= + +/** + * Options for kicking off an asynchronous `$sqlquery-export` operation. + */ +export interface SqlQueryExportKickOffOptions extends AuthOptions { + /** The export request, reusing the synchronous run's query source. */ + request: SqlQueryExportRequest; +} + +/** + * Result of a `$sqlquery-export` kick-off: the polling URL from the {@code Content-Location} + * header. + */ +export interface SqlQueryExportResult { + pollingUrl: string; +} + +/** + * Options for downloading a `$sqlquery-export` output file. + */ +export interface SqlQueryExportDownloadOptions extends AuthOptions { + /** The fully-qualified download URL from the manifest's `output.location`. */ + location: string; +} + +/** + * Kicks off an asynchronous `$sqlquery-export` operation. + * + * @param baseUrl - The FHIR server base URL. + * @param options - The export options including the query source, format, and header flag. + * @returns The polling URL for tracking the export operation. + * @throws {UnauthorizedError} When the request receives a 401 response. + * @throws {Error} For other non-successful responses or a missing Content-Location header. + * + * @example + * const { pollingUrl } = await sqlQueryExportKickOff("https://example.com/fhir", { + * request: { mode: "stored", libraryId: "patient-bp-query", format: "ndjson" }, + * accessToken: "token123", + * }); + */ +export async function sqlQueryExportKickOff( + baseUrl: string, + options: SqlQueryExportKickOffOptions, +): Promise { + const url = buildUrl(baseUrl, "/$sqlquery-export"); + const headers = buildHeaders({ + accessToken: options.accessToken, + contentType: "application/fhir+json", + prefer: "respond-async", + }); + + const body = buildSqlQueryExportKickOffBody(options.request); + + const response = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(body), + }); + + await checkResponse(response, "SQL query export kick-off"); + + if (response.status !== 202) { + const errorBody = await response.text(); + throw new Error( + `SQL query export kick-off failed: ${response.status} - ${errorBody}`, + ); + } + + const contentLocation = response.headers.get("Content-Location"); + if (!contentLocation) { + throw new Error( + "SQL query export kick-off failed: No Content-Location header", + ); + } + + return { pollingUrl: contentLocation }; +} + +/** + * Downloads a `$sqlquery-export` output file from its manifest location URL. + * + * @param options - Download options including the fully-qualified location URL. + * @returns A ReadableStream of the file contents. + * @throws {UnauthorizedError} When the request receives a 401 response. + * @throws {Error} For other non-successful responses or a missing body. + * + * @example + * const stream = await sqlQueryExportDownload({ + * location: "https://example.com/fhir/$result?job=abc&file=people.ndjson", + * accessToken: "token123", + * }); + */ +export async function sqlQueryExportDownload( + options: SqlQueryExportDownloadOptions, +): Promise { + const headers = buildHeaders({ accessToken: options.accessToken }); + + const response = await fetch(options.location, { method: "GET", headers }); + + await checkResponse(response, "SQL query export download"); + + if (!response.body) { + throw new Error("SQL query export download failed: No response body"); + } + + return response.body; +} diff --git a/ui/src/components/sqlOnFhir/ExportJobCard.tsx b/ui/src/components/sqlOnFhir/ExportJobCard.tsx new file mode 100644 index 0000000000..9bdb77f1cf --- /dev/null +++ b/ui/src/components/sqlOnFhir/ExportJobCard.tsx @@ -0,0 +1,262 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Reusable, presentational card displaying an asynchronous export job (view export or SQL query + * export) with progress, cancellation, and download links. All state management happens in the + * parent; this component is parameterised by a manifest-output parser so it can serve either export + * operation, both of which share the SQL on FHIR manifest shape and the NDJSON/CSV/Parquet format + * set. + * + * @author John Grimes + */ + +import { Cross2Icon, DownloadIcon, ReloadIcon, TrashIcon } from "@radix-ui/react-icons"; +import { Badge, Box, Button, Card, Flex, Progress, Text } from "@radix-ui/themes"; +import { useState } from "react"; + +import { OperationOutcomeDisplay } from "../../components/error/OperationOutcomeDisplay"; +import { formatDateTime } from "../../utils"; + +import type { JobStatus } from "../../types/job"; +import type { Parameters } from "fhir/r4"; + +/** The NDJSON/CSV/Parquet format set shared by both export operations. */ +export type ExportJobFormat = "ndjson" | "csv" | "parquet"; + +/** One downloadable output extracted from a completion manifest. */ +export interface ExportJobOutput { + name: string; + url: string; +} + +/** The presentational state of an export job, common to view and SQL query export. */ +export interface ExportJobCardData { + status: JobStatus; + progress: number | null; + error: Error | null; + format: ExportJobFormat; + manifest: Parameters | null; + createdAt: Date; +} + +interface ExportJobCardProps { + job: ExportJobCardData; + /** Parses the completion manifest into the downloadable outputs. */ + getOutputs: (manifest: Parameters) => ExportJobOutput[]; + onCancel: () => void; + onDownload: (url: string, filename: string) => void; + onClose?: () => void; + /** Optional callback to delete the export files from the server. */ + onDelete?: () => Promise; +} + +const STATUS_COLORS: Record = { + pending: "blue", + in_progress: "blue", + completed: "green", + failed: "red", + cancelled: "gray", +}; + +const STATUS_LABELS: Record = { + pending: "Pending", + in_progress: "Exporting", + completed: "Completed", + failed: "Failed", + cancelled: "Cancelled", +}; + +const FORMAT_LABELS: Record = { + ndjson: "NDJSON", + csv: "CSV", + parquet: "Parquet", +}; + +/** + * Extracts the filename from a result URL's query parameters. + * + * @param url - The URL to extract the filename from. + * @returns The extracted filename or "unknown" if not found. + */ +function getFilenameFromUrl(url: string): string { + const params = new URLSearchParams(new URL(url).search); + return params.get("file") ?? "unknown"; +} + +/** + * Displays an export job's status with progress and download links. + * + * @param root0 - The component props. + * @param root0.job - The export job state to display. + * @param root0.getOutputs - Parses the manifest into downloadable outputs. + * @param root0.onCancel - Callback to cancel the export. + * @param root0.onDownload - Callback to download an output file. + * @param root0.onClose - Optional callback to close/remove the card. + * @param root0.onDelete - Optional callback to delete export files from the server. + * @returns The export job card component. + */ +export function ExportJobCard({ + job, + getOutputs, + onCancel, + onDownload, + onClose, + onDelete, +}: Readonly) { + const [isDeleting, setIsDeleting] = useState(false); + + const isActive = job.status === "pending" || job.status === "in_progress"; + const showProgress = isActive && job.progress !== null; + const canDelete = job.status === "completed" && onDelete !== undefined; + const canClose = + job.status === "completed" || job.status === "cancelled" || job.status === "failed"; + + /** + * Handles the delete action by cleaning up server files, then closing the card. + */ + async function handleDelete() { + if (!onDelete) return; + setIsDeleting(true); + try { + await onDelete(); + } finally { + setIsDeleting(false); + onClose?.(); + } + } + + return ( + + + + + + + Export to {FORMAT_LABELS[job.format]} + + + {STATUS_LABELS[job.status]} + + + + {formatDateTime(job.createdAt)} + + + {isActive && ( + + )} + {canDelete && onClose && ( + + + + + )} + {canClose && !canDelete && onClose && ( + + )} + + + {showProgress && ( + + + + Progress + + + {job.progress}% + + + + + )} + + {isActive && !showProgress && ( + + + + Exporting... + + + )} + + {job.status === "failed" && job.error && } + + {job.status === "completed" && + job.manifest && + (() => { + const outputs = getOutputs(job.manifest); + return outputs.length > 0 ? ( + + + Output files ({outputs.length}) + + + {outputs + .slice() + .sort((a, b) => + getFilenameFromUrl(a.url).localeCompare(getFilenameFromUrl(b.url)), + ) + .map((output) => ( + + + {getFilenameFromUrl(output.url)} + + + + ))} + + + ) : null; + })()} + + + ); +} diff --git a/ui/src/components/sqlOnFhir/SqlQueryCard.tsx b/ui/src/components/sqlOnFhir/SqlQueryCard.tsx index 0e552360db..8067c05418 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryCard.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryCard.tsx @@ -34,17 +34,27 @@ import { Card, Code, Flex, + Separator, Spinner, Table, Text, } from "@radix-ui/themes"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; +import { ExportControls } from "./ExportControls"; +import { SqlQueryExportCardWrapper } from "./SqlQueryExportCardWrapper"; import { useSqlQueryRun } from "../../hooks"; import { OperationOutcomeError } from "../../types/errors"; import { formatDateTime } from "../../utils"; -import type { SqlQueryJob, SqlQueryResult } from "../../types/sqlQuery"; +import type { SqlQueryExportFormat, SqlQueryJob, SqlQueryResult } from "../../types/sqlQuery"; + +/** An in-progress or completed export spawned from this result card. */ +interface SqlQueryExportEntry { + id: string; + format: SqlQueryExportFormat; + createdAt: Date; +} interface SqlQueryCardProps { /** The SQL query job describing the request. */ @@ -66,12 +76,19 @@ interface SqlQueryCardProps { */ export function SqlQueryCard({ job, onError, onClose }: Readonly) { const { execute, status, result, error } = useSqlQueryRun(); + const [exports, setExports] = useState([]); const isRunning = status === "pending"; const isComplete = status === "success"; const isError = status === "error"; const canClose = isComplete || isError; + // The export affordance appears once a run has returned data: rows for a tabular result, or a + // file for a non-previewable binary (Parquet) result. + const hasRows = + result !== undefined && + (result.kind === "binary" || (result.kind === "tabular" && result.rows.length > 0)); + // Mount-time execution: kick off the request once when the card lands // in idle state. Using the status as the trigger plays nicely with React // Strict Mode's double-render of the mount. @@ -88,6 +105,27 @@ export function SqlQueryCard({ job, onError, onClose }: Readonly [ + ...current, + { id: crypto.randomUUID(), format, createdAt: new Date() }, + ]); + } + + /** + * Removes an export card. + * + * @param id - The id of the export to remove. + */ + function handleCloseExport(id: string) { + setExports((current) => current.filter((entry) => entry.id !== id)); + } + return ( @@ -134,6 +172,30 @@ export function SqlQueryCard({ job, onError, onClose }: Readonly} {isComplete && result && } + + {isComplete && hasRows && ( + <> + + + + + Export full result set + + + + {exports.map((entry) => ( + handleCloseExport(entry.id)} + onError={onError} + /> + ))} + + + )} ); @@ -157,8 +219,8 @@ function SqlQueryResultBody({ result, sql }: Readonly) if (result.kind === "binary") { return ( - Binary results cannot be previewed. Download will be supported by a future SQL query export - operation. + Parquet results cannot be previewed. Use the Export control below to download the full + result set. ); } diff --git a/ui/src/components/sqlOnFhir/SqlQueryExportCardWrapper.tsx b/ui/src/components/sqlOnFhir/SqlQueryExportCardWrapper.tsx new file mode 100644 index 0000000000..21ce4f35ac --- /dev/null +++ b/ui/src/components/sqlOnFhir/SqlQueryExportCardWrapper.tsx @@ -0,0 +1,136 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Wrapper component that manages a single SQL query export job lifecycle. Starts the export on + * mount, reusing the synchronous run's query source, and renders the reusable {@link ExportJobCard} + * with the current state. + * + * @author John Grimes + */ + +import { useEffect, useRef } from "react"; + +import { ExportJobCard } from "./ExportJobCard"; +import { parseSqlQueryExportManifest, useDownloadFile, useSqlQueryExport } from "../../hooks"; + +import type { JobStatus } from "../../types/job"; +import type { + SqlQueryExportFormat, + SqlQueryExportManifest, + SqlQueryExportRequest, + SqlQueryRequest, +} from "../../types/sqlQuery"; +import type { Parameters } from "fhir/r4"; + +interface SqlQueryExportCardWrapperProps { + /** The synchronous run's query source, reused for the export. */ + source: SqlQueryRequest; + format: SqlQueryExportFormat; + createdAt: Date; + onClose: () => void; + onError: (message: string) => void; +} + +/** + * Derives the export request from the synchronous run's query source and the chosen format. + * + * @param source - The synchronous run's request (stored or inline). + * @param format - The chosen export format. + * @returns The export request. + */ +function toExportRequest( + source: SqlQueryRequest, + format: SqlQueryExportFormat, +): SqlQueryExportRequest { + return source.mode === "stored" + ? { mode: "stored", libraryId: source.libraryId, format, header: true } + : { mode: "inline", library: source.library, format, header: true }; +} + +/** + * Maps the async-job status to the export job card status. + * + * @param status - The async-job status. + * @returns The corresponding job card status. + */ +function toJobStatus(status: string): JobStatus { + switch (status) { + case "pending": + case "in-progress": + return "in_progress"; + case "complete": + return "completed"; + case "error": + return "failed"; + case "cancelled": + return "cancelled"; + default: + return "pending"; + } +} + +/** + * Manages a SQL query export job lifecycle, starting it on mount. + * + * @param props - Component props. + * @param props.source - The synchronous run's query source, reused for the export. + * @param props.format - The output format for the export. + * @param props.createdAt - The timestamp when the export was created. + * @param props.onClose - Callback to remove this export card. + * @param props.onError - Callback for error handling. + * @returns The rendered export card. + */ +export function SqlQueryExportCardWrapper({ + source, + format, + createdAt, + onClose, + onError, +}: Readonly) { + const hasStartedRef = useRef(false); + const handleDownload = useDownloadFile((err) => onError(err.message)); + + const { startWith, cancel, status, result, error, progress } = useSqlQueryExport({ + onError: (err) => onError(err.message), + }); + + // Start the export on mount. + useEffect(() => { + if (!hasStartedRef.current) { + hasStartedRef.current = true; + startWith(toExportRequest(source, format)); + } + }, [source, format, startWith]); + + return ( + parseSqlQueryExportManifest(manifest)} + onCancel={cancel} + onDownload={handleDownload} + onClose={onClose} + /> + ); +} diff --git a/ui/src/components/sqlOnFhir/ViewExportCard.tsx b/ui/src/components/sqlOnFhir/ViewExportCard.tsx index 6a1356c97b..82daab879a 100644 --- a/ui/src/components/sqlOnFhir/ViewExportCard.tsx +++ b/ui/src/components/sqlOnFhir/ViewExportCard.tsx @@ -16,22 +16,16 @@ */ /** - * Card component displaying a view export job with progress and download links. - * This is a pure presentational component - all state management happens in the parent. + * Card component displaying a view export job. A thin adapter over the reusable {@link + * ExportJobCard}, supplying the view export manifest parser. * * @author John Grimes */ -import { Cross2Icon, DownloadIcon, ReloadIcon, TrashIcon } from "@radix-ui/react-icons"; -import { Badge, Box, Button, Card, Flex, Progress, Text } from "@radix-ui/themes"; -import { useState } from "react"; - -import { OperationOutcomeDisplay } from "../../components/error/OperationOutcomeDisplay"; +import { ExportJobCard } from "./ExportJobCard"; import { getViewExportOutputFiles } from "../../types/viewExport"; -import { formatDateTime } from "../../utils"; import type { ViewExportJob } from "../../types/job"; -import type { ViewExportFormat } from "../../types/viewExport"; interface ViewExportCardProps { job: ViewExportJob; @@ -42,41 +36,8 @@ interface ViewExportCardProps { onDelete?: () => Promise; } -const STATUS_COLORS: Record = { - pending: "blue", - in_progress: "blue", - completed: "green", - failed: "red", - cancelled: "gray", -}; - -const STATUS_LABELS: Record = { - pending: "Pending", - in_progress: "Exporting", - completed: "Completed", - failed: "Failed", - cancelled: "Cancelled", -}; - -const FORMAT_LABELS: Record = { - ndjson: "NDJSON", - csv: "CSV", - parquet: "Parquet", -}; - -/** - * Extracts the filename from a result URL's query parameters. - * - * @param url - The URL to extract the filename from. - * @returns The extracted filename or "unknown" if not found. - */ -function getFilenameFromUrl(url: string): string { - const params = new URLSearchParams(new URL(url).search); - return params.get("file") ?? "unknown"; -} - /** - * Displays export job status with progress and download links. + * Displays a view export job's status with progress and download links. * * @param root0 - The component props. * @param root0.job - The view export job to display. @@ -93,146 +54,21 @@ export function ViewExportCard({ onClose, onDelete, }: Readonly) { - const [isDeleting, setIsDeleting] = useState(false); - - const isActive = job.status === "pending" || job.status === "in_progress"; - const showProgress = isActive && job.progress !== null; - const canDelete = job.status === "completed" && onDelete !== undefined; - const canClose = - job.status === "completed" || job.status === "cancelled" || job.status === "failed"; - - /** - * Handles the delete action by calling onDelete to clean up server files, - * then closing the card. - */ - async function handleDelete() { - if (!onDelete) return; - setIsDeleting(true); - try { - await onDelete(); - } finally { - setIsDeleting(false); - onClose?.(); - } - } - return ( - - - - - - - Export to {FORMAT_LABELS[job.request.format]} - - - {STATUS_LABELS[job.status]} - - - - {formatDateTime(job.createdAt)} - - - {isActive && ( - - )} - {canDelete && onClose && ( - - - - - )} - {canClose && !canDelete && onClose && ( - - )} - - - {showProgress && ( - - - - Progress - - - {job.progress}% - - - - - )} - - {isActive && !showProgress && ( - - - - Exporting... - - - )} - - {job.status === "failed" && job.error && } - - {job.status === "completed" && - job.manifest && - (() => { - const outputs = getViewExportOutputFiles(job.manifest); - return outputs.length > 0 ? ( - - - Output files ({outputs.length}) - - - {outputs - .slice() - .sort((a, b) => - getFilenameFromUrl(a.url).localeCompare(getFilenameFromUrl(b.url)), - ) - .map((output) => ( - - - {getFilenameFromUrl(output.url)} - - - - ))} - - - ) : null; - })()} - - + ); } diff --git a/ui/src/components/sqlOnFhir/__tests__/SqlQueryCard.test.tsx b/ui/src/components/sqlOnFhir/__tests__/SqlQueryCard.test.tsx index bd844edd88..7f62dcfda2 100644 --- a/ui/src/components/sqlOnFhir/__tests__/SqlQueryCard.test.tsx +++ b/ui/src/components/sqlOnFhir/__tests__/SqlQueryCard.test.tsx @@ -159,14 +159,14 @@ describe("SqlQueryCard", () => { expect(screen.queryByText("pat-11")).not.toBeInTheDocument(); }); - // The format indicator (e.g. "ndjson") is not rendered alongside the - // close button - format selection is owned by the form. - it("does not render a format badge in the header", () => { + // Once a run returns rows, the result card offers an export affordance: a + // format picker and an Export button to start an asynchronous export. + it("renders export controls once the run returns rows", () => { mockStatus = "success"; mockResult = TABULAR_RESULT; render(); - expect(screen.queryByText(/^csv$/i)).not.toBeInTheDocument(); - expect(screen.queryByText(/^ndjson$/i)).not.toBeInTheDocument(); + expect(screen.getByText(/export full result set/i)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /^export$/i })).toBeInTheDocument(); }); // Empty tabular results show "No rows returned" instead of an empty @@ -181,10 +181,9 @@ describe("SqlQueryCard", () => { expect(screen.getByText(/no rows returned/i)).toBeInTheDocument(); }); - // Binary (parquet) results cannot be previewed in the card; the body - // explains that downloads will be supported by a future export - // operation. - it("renders an export-pending notice for parquet results", () => { + // Binary (parquet) results cannot be previewed in the card; the body points + // the operator to the Export control to download the full result set. + it("offers export for non-previewable parquet results", () => { mockStatus = "success"; mockResult = BINARY_RESULT; render( @@ -196,9 +195,10 @@ describe("SqlQueryCard", () => { onClose={onClose} />, ); - expect(screen.getByText(/binary results cannot be previewed/i)).toBeInTheDocument(); - expect(screen.queryByRole("button", { name: /download/i })).not.toBeInTheDocument(); + expect(screen.getByText(/parquet results cannot be previewed/i)).toBeInTheDocument(); + // No preview table for a binary result, but the export affordance is present. expect(screen.queryByRole("table")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: /^export$/i })).toBeInTheDocument(); }); // OperationOutcome errors are shown in a callout with the submitted SQL diff --git a/ui/src/hooks/__tests__/sqlQueryExportHelpers.test.ts b/ui/src/hooks/__tests__/sqlQueryExportHelpers.test.ts new file mode 100644 index 0000000000..ae0154422e --- /dev/null +++ b/ui/src/hooks/__tests__/sqlQueryExportHelpers.test.ts @@ -0,0 +1,141 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, it } from "vitest"; + +import { + buildSqlQueryExportKickOffBody, + parseSqlQueryExportManifest, +} from "../sqlQueryExportHelpers"; + +import type { SqlQueryLibrary } from "../../types/sqlQuery"; +import type { Parameters } from "fhir/r4"; + +describe("buildSqlQueryExportKickOffBody", () => { + it("builds a query.queryReference part for a stored query", () => { + const body = buildSqlQueryExportKickOffBody({ + mode: "stored", + libraryId: "patient-bp-query", + format: "ndjson", + }); + + expect(body.resourceType).toBe("Parameters"); + const query = body.parameter?.find((p) => p.name === "query"); + expect(query?.part?.[0]?.name).toBe("queryReference"); + expect(query?.part?.[0]?.valueReference?.reference).toBe( + "Library/patient-bp-query", + ); + expect(body.parameter?.find((p) => p.name === "_format")?.valueCode).toBe( + "ndjson", + ); + }); + + it("builds a query.queryResource part for an inline query", () => { + const library: SqlQueryLibrary = { + resourceType: "Library", + status: "active", + type: { + coding: [ + { + system: "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes", + code: "sql-query", + }, + ], + }, + content: [{ contentType: "application/sql", data: "U0VMRUNUIDE=" }], + }; + + const body = buildSqlQueryExportKickOffBody({ + mode: "inline", + library, + format: "csv", + header: false, + }); + + const query = body.parameter?.find((p) => p.name === "query"); + expect(query?.part?.[0]?.name).toBe("queryResource"); + expect(query?.part?.[0]?.resource?.resourceType).toBe("Library"); + expect(body.parameter?.find((p) => p.name === "_format")?.valueCode).toBe( + "csv", + ); + expect(body.parameter?.find((p) => p.name === "header")?.valueBoolean).toBe( + false, + ); + }); + + it("omits the header parameter when not supplied", () => { + const body = buildSqlQueryExportKickOffBody({ + mode: "stored", + libraryId: "q", + format: "parquet", + }); + expect(body.parameter?.find((p) => p.name === "header")).toBeUndefined(); + }); +}); + +describe("parseSqlQueryExportManifest", () => { + it("returns an empty array for an absent manifest or parameters", () => { + expect(parseSqlQueryExportManifest(null)).toEqual([]); + expect(parseSqlQueryExportManifest(undefined)).toEqual([]); + expect(parseSqlQueryExportManifest({ resourceType: "Parameters" })).toEqual( + [], + ); + }); + + it("extracts one entry per location, sharing the output name", () => { + const manifest: Parameters = { + resourceType: "Parameters", + parameter: [ + { name: "status", valueCode: "completed" }, + { + name: "output", + part: [ + { name: "name", valueString: "people" }, + { + name: "location", + valueUri: "https://x/fhir/$result?job=j&file=people.00000.ndjson", + }, + { + name: "location", + valueUri: "https://x/fhir/$result?job=j&file=people.00001.ndjson", + }, + ], + }, + { + name: "output", + part: [ + { name: "name", valueString: "observations" }, + { + name: "location", + valueUri: + "https://x/fhir/$result?job=j&file=observations.00000.ndjson", + }, + ], + }, + ], + }; + + const outputs = parseSqlQueryExportManifest(manifest); + expect(outputs).toHaveLength(3); + expect(outputs[0]).toEqual({ + name: "people", + url: "https://x/fhir/$result?job=j&file=people.00000.ndjson", + }); + expect(outputs[1].name).toBe("people"); + expect(outputs[2].name).toBe("observations"); + }); +}); diff --git a/ui/src/hooks/index.ts b/ui/src/hooks/index.ts index 1e0523532b..0fa1579774 100644 --- a/ui/src/hooks/index.ts +++ b/ui/src/hooks/index.ts @@ -35,6 +35,16 @@ export { useBulkSubmit } from "./useBulkSubmit"; export { useBulkSubmit as useBulkSubmitMonitor } from "./useBulkSubmit"; export { useViewExport } from "./useViewExport"; export type { ViewExportOutputFormat } from "./useViewExport"; +export { useSqlQueryExport } from "./useSqlQueryExport"; +export type { + UseSqlQueryExportOptions, + UseSqlQueryExportResult, +} from "./useSqlQueryExport"; +export { + buildSqlQueryExportKickOffBody, + parseSqlQueryExportManifest, +} from "./sqlQueryExportHelpers"; +export type { SqlQueryExportOutput } from "./sqlQueryExportHelpers"; // View operations. export { useViewRun } from "./useViewRun"; diff --git a/ui/src/hooks/sqlQueryExportHelpers.ts b/ui/src/hooks/sqlQueryExportHelpers.ts new file mode 100644 index 0000000000..714e6bd992 --- /dev/null +++ b/ui/src/hooks/sqlQueryExportHelpers.ts @@ -0,0 +1,118 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Pure helpers for the `$sqlquery-export` UI flow: building the kick-off request body and parsing + * the completion manifest. Kept as plain functions so they can be unit tested without React. + * + * @author John Grimes + */ + +import type { SqlQueryExportRequest } from "../types/sqlQuery"; +import type { Parameters, ParametersParameter } from "fhir/r4"; + +/** + * One downloadable output extracted from a `$sqlquery-export` completion manifest. + */ +export interface SqlQueryExportOutput { + /** The output name. */ + name: string; + /** The download URL (a `$result` URL from the manifest's `location`). */ + url: string; +} + +/** + * Builds the `$sqlquery-export` kick-off request body, reusing the same query source (stored or + * inline) as the synchronous run. + * + * @param request - The export request describing the query source, format, and header flag. + * @returns A FHIR Parameters resource for the kick-off request body. + * + * @example + * const body = buildSqlQueryExportKickOffBody({ + * mode: "stored", + * libraryId: "patient-bp-query", + * format: "ndjson", + * }); + */ +export function buildSqlQueryExportKickOffBody( + request: SqlQueryExportRequest, +): Parameters { + const queryPart: ParametersParameter = + request.mode === "stored" + ? { + name: "query", + part: [ + { + name: "queryReference", + valueReference: { reference: `Library/${request.libraryId}` }, + }, + ], + } + : { + name: "query", + part: [ + { + name: "queryResource", + resource: + request.library as Parameters["parameter"] extends (infer T)[] + ? T extends { resource?: infer R } + ? R + : never + : never, + }, + ], + }; + + const parameter: ParametersParameter[] = [ + queryPart, + { name: "_format", valueCode: request.format }, + ]; + + if (request.header !== undefined) { + parameter.push({ name: "header", valueBoolean: request.header }); + } + + return { resourceType: "Parameters", parameter }; +} + +/** + * Extracts the downloadable outputs from a `$sqlquery-export` completion manifest. + * + * Each SQL on FHIR `output` parameter carries one `name` part and one or more `location` parts (one + * per file the query produced). This emits one entry per `location`, all sharing the output's name, + * so a query that produced several files lists every file. + * + * @param manifest - The completion manifest Parameters resource (or null/undefined). + * @returns The output entries, one per file, in manifest order. + */ +export function parseSqlQueryExportManifest( + manifest: Parameters | null | undefined, +): SqlQueryExportOutput[] { + if (!manifest?.parameter) { + return []; + } + return manifest.parameter + .filter((param) => param.name === "output" && param.part) + .flatMap((param) => { + const parts = param.part!; + const name = parts.find((p) => p.name === "name")?.valueString ?? ""; + return parts + .filter((p) => p.name === "location" && p.valueUri) + .map((p) => ({ name, url: p.valueUri! })); + }); +} diff --git a/ui/src/hooks/useSqlQueryExport.ts b/ui/src/hooks/useSqlQueryExport.ts new file mode 100644 index 0000000000..c10dc6b678 --- /dev/null +++ b/ui/src/hooks/useSqlQueryExport.ts @@ -0,0 +1,99 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useCallback, useRef } from "react"; + +import { + jobCancel, + jobStatus, + sqlQueryExportDownload, + sqlQueryExportKickOff, +} from "../api"; +import { config } from "../config"; +import { useAsyncJob } from "./useAsyncJob"; +import { useAsyncJobCallbacks } from "./useAsyncJobCallbacks"; +import { useAuth } from "../contexts/AuthContext"; + +import type { AsyncJobOptions, UseAsyncJobResult } from "./useAsyncJob"; +import type { + SqlQueryExportManifest, + SqlQueryExportRequest, +} from "../types/sqlQuery"; + +/** + * Options for the {@link useSqlQueryExport} hook (callbacks only). + */ +export type UseSqlQueryExportOptions = AsyncJobOptions; + +/** + * Result of the {@link useSqlQueryExport} hook. + */ +export interface UseSqlQueryExportResult extends UseAsyncJobResult< + SqlQueryExportRequest, + SqlQueryExportManifest +> { + /** Downloads an output file from the manifest by its location URL. */ + download: (location: string) => Promise; +} + +/** + * Executes a `$sqlquery-export` operation with polling, reusing the synchronous run's query source. + * A thin wrapper composing {@link useAsyncJob} with the export kick-off, status polling, and cancel + * APIs. + * + * @param options - Optional callbacks for progress, completion, and error events. + * @returns Hook result with status, result, control functions, and a download function. + */ +export function useSqlQueryExport( + options?: UseSqlQueryExportOptions, +): UseSqlQueryExportResult { + const { fhirBaseUrl } = config; + const { client } = useAuth(); + const accessToken = client?.state.tokenResponse?.access_token; + const pollingUrlRef = useRef(undefined); + + const callbacks = useAsyncJobCallbacks(options); + + const buildOptions = useCallback( + (request: SqlQueryExportRequest) => ({ + kickOff: () => + sqlQueryExportKickOff(fhirBaseUrl!, { request, accessToken }), + getJobId: (result: { pollingUrl: string }) => { + pollingUrlRef.current = result.pollingUrl; + return result.pollingUrl; + }, + checkStatus: (pollingUrl: string) => + jobStatus(fhirBaseUrl!, { pollingUrl, accessToken }), + isComplete: (status: { status: string }) => status.status === "complete", + getResult: (status: { result?: unknown }) => + status.result as UseSqlQueryExportResult["result"], + cancel: (pollingUrl: string) => + jobCancel(fhirBaseUrl!, { pollingUrl, accessToken }), + pollingInterval: 3000, + }), + [fhirBaseUrl, accessToken], + ); + + const job = useAsyncJob(buildOptions, callbacks); + + const download = useCallback( + (location: string) => sqlQueryExportDownload({ location, accessToken }), + [accessToken], + ); + + return { ...job, download }; +} diff --git a/ui/src/types/job.ts b/ui/src/types/job.ts index 723785f842..7473414760 100644 --- a/ui/src/types/job.ts +++ b/ui/src/types/job.ts @@ -29,6 +29,7 @@ import type { import type { ExportRequest, ExportManifest } from "./export"; import type { ImportRequest, ImportManifest } from "./import"; import type { ImportPnpRequest } from "./importPnp"; +import type { SqlQueryExportFormat, SqlQueryExportManifest } from "./sqlQuery"; import type { ViewExportRequest, ViewExportManifest } from "./viewExport"; export type JobType = @@ -36,7 +37,8 @@ export type JobType = | "import" | "import-pnp" | "bulk-submit" - | "view-export"; + | "view-export" + | "sqlquery-export"; export type JobStatus = | "pending" @@ -87,9 +89,16 @@ export interface ViewExportJob extends BaseJob { manifest: ViewExportManifest | null; } +export interface SqlQueryExportJob extends BaseJob { + type: "sqlquery-export"; + request: { format: SqlQueryExportFormat }; + manifest: SqlQueryExportManifest | null; +} + export type Job = | ExportJob | ImportJob | ImportPnpJob | BulkSubmitJob - | ViewExportJob; + | ViewExportJob + | SqlQueryExportJob; diff --git a/ui/src/types/sqlQuery.ts b/ui/src/types/sqlQuery.ts index e5173fc0cd..56f03a05a1 100644 --- a/ui/src/types/sqlQuery.ts +++ b/ui/src/types/sqlQuery.ts @@ -227,3 +227,37 @@ export interface SaveSqlQueryLibraryResult { id: string; title: string; } + +/** + * Output formats accepted by the asynchronous `$sqlquery-export` operation. + * + * Narrower than {@link SqlQueryOutputFormat}: the export operation supports only the file-friendly + * formats, mirroring `$viewdefinition-export`. + */ +export type SqlQueryExportFormat = "ndjson" | "csv" | "parquet"; + +/** + * A `$sqlquery-export` request, reusing the same query source (stored or inline) as the + * synchronous run, plus the chosen export format and CSV header flag. + */ +export type SqlQueryExportRequest = + | { + mode: "stored"; + /** ID of a stored Library conforming to the SQLQuery profile. */ + libraryId: string; + format: SqlQueryExportFormat; + header?: boolean; + } + | { + mode: "inline"; + /** Inline Library to send as the `query.queryResource` part. */ + library: SqlQueryLibrary; + format: SqlQueryExportFormat; + header?: boolean; + }; + +/** + * The `$sqlquery-export` completion manifest, a FHIR Parameters resource describing the export + * outputs. Shares the SQL on FHIR manifest shape with the view export manifest. + */ +export type SqlQueryExportManifest = import("fhir/r4").Parameters; From f76769e950d4f3477c8495009f2c799657786887 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 08:22:13 +1000 Subject: [PATCH 011/162] refactor: Name the async wire contract with an AsyncPattern enum Replace the boolean redirectOnComplete marker on the server's asynchronous machinery with an AsyncPattern enum (STANDARD_ASYNC_PATTERN, BULK_DATA), so the @AsyncSupported annotation reads as a deliberate choice between two named patterns rather than the presence or absence of one behaviour. Correct the Javadoc that misattributed the redirect-based completion to a "SQL on FHIR unify-async" specification, citing the HL7 Asynchronous Interaction Request Pattern instead. Behaviour is unchanged; the existing async tests pass with only their identifiers updated. --- .../au/csiro/pathling/async/AsyncAspect.java | 11 +++-- .../au/csiro/pathling/async/AsyncPattern.java | 47 +++++++++++++++++++ .../csiro/pathling/async/AsyncSupported.java | 11 +++-- .../java/au/csiro/pathling/async/Job.java | 9 ++-- .../au/csiro/pathling/async/JobProvider.java | 13 +++-- .../pathling/async/JobResultProvider.java | 5 +- .../sqlquery/SqlQueryExportProvider.java | 3 +- .../SqlQueryInstanceExportProvider.java | 5 +- .../view/ViewDefinitionExportProvider.java | 3 +- .../csiro/pathling/async/JobProviderTest.java | 17 ++++--- .../pathling/async/JobResultProviderTest.java | 20 ++++---- .../view/ViewDefinitionExportProviderIT.java | 4 +- 12 files changed, 104 insertions(+), 44 deletions(-) create mode 100644 server/src/main/java/au/csiro/pathling/async/AsyncPattern.java diff --git a/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java b/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java index 3a22c5b256..158d2fd215 100644 --- a/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java +++ b/server/src/main/java/au/csiro/pathling/async/AsyncAspect.java @@ -141,10 +141,11 @@ protected IBaseResource maybeExecuteAsynchronously( final Job job = processRequestAsynchronously(joinPoint, requestDetails, result, spark, asyncSupported); - if (asyncSupported.redirectOnComplete()) { - // For SQL on FHIR async operations, the kick-off body is a Parameters acknowledgement - // (status=accepted, exportId) returned with a 202 status, rather than an OperationOutcome. - // The Content-Location header is already set by processRequestAsynchronously. + if (asyncSupported.pattern() == AsyncPattern.STANDARD_ASYNC_PATTERN) { + // Under the HL7 Asynchronous Interaction Request Pattern, the kick-off body is a Parameters + // acknowledgement (status=accepted, exportId) returned with a 202 status, rather than an + // OperationOutcome. The Content-Location header is already set by + // processRequestAsynchronously. final HttpServletResponse response = requestDetails.getServletResponse(); if (response != null) { response.setStatus(Constants.STATUS_HTTP_202_ACCEPTED); @@ -235,7 +236,7 @@ private Job processRequestAsynchronously( final Optional ownerId = getCurrentUserId(authentication); final Job newJob = new Job<>(jobId, operation, result, ownerId); newJob.setPreAsyncValidationResult(preAsyncValidationResult.result()); - newJob.setRedirectOnComplete(asyncSupported.redirectOnComplete()); + newJob.setPattern(asyncSupported.pattern()); return newJob; }); final HttpServletResponse response = requestDetails.getServletResponse(); diff --git a/server/src/main/java/au/csiro/pathling/async/AsyncPattern.java b/server/src/main/java/au/csiro/pathling/async/AsyncPattern.java new file mode 100644 index 0000000000..e9582194a0 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/async/AsyncPattern.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.async; + +/** + * The asynchronous wire contract that an operation follows. The two patterns differ in the kick-off + * acknowledgement body returned with the {@code 202 Accepted} response and in how a completed job + * is delivered when polled via {@code $job}. + * + *

    The redirect-based contract is the HL7 Asynchronous Interaction Request Pattern, defined by + * the HL7 R6 API Incubator: Asynchronous + * Interaction Request Pattern. + * + * @author John Grimes + */ +public enum AsyncPattern { + + /** + * The HL7 Asynchronous Interaction Request Pattern. Kick-off returns a {@code Parameters} + * acknowledgement (with {@code status=accepted} and an {@code exportId}), and a completed job + * returns {@code 303 See Other} pointing at a separate {@code $job-result} endpoint. + */ + STANDARD_ASYNC_PATTERN, + + /** + * The FHIR Bulk Data pattern used by {@code $export}/{@code $import}. Kick-off returns an {@code + * OperationOutcome}, and a completed job returns the result manifest inline with {@code 200 OK}. + * This is the default. + */ + BULK_DATA +} diff --git a/server/src/main/java/au/csiro/pathling/async/AsyncSupported.java b/server/src/main/java/au/csiro/pathling/async/AsyncSupported.java index ab52a4ad33..3f6f37318d 100644 --- a/server/src/main/java/au/csiro/pathling/async/AsyncSupported.java +++ b/server/src/main/java/au/csiro/pathling/async/AsyncSupported.java @@ -36,10 +36,13 @@ public @interface AsyncSupported { /** - * When true, completed jobs return 303 See Other with a redirect to the result endpoint, rather - * than returning the result inline. This follows the SQL on FHIR unify-async specification. + * The asynchronous wire contract this operation follows. Selecting {@link + * AsyncPattern#STANDARD_ASYNC_PATTERN} (the HL7 Asynchronous Interaction Request Pattern, spec) + * makes a completed job return 303 See Other with a redirect to the result endpoint, rather than + * returning the result inline. Defaults to {@link AsyncPattern#BULK_DATA}. * - * @return true if completed jobs should redirect to the result endpoint + * @return the asynchronous pattern for this operation */ - boolean redirectOnComplete() default false; + AsyncPattern pattern() default AsyncPattern.BULK_DATA; } diff --git a/server/src/main/java/au/csiro/pathling/async/Job.java b/server/src/main/java/au/csiro/pathling/async/Job.java index 2c853abcbe..04d6f31310 100644 --- a/server/src/main/java/au/csiro/pathling/async/Job.java +++ b/server/src/main/java/au/csiro/pathling/async/Job.java @@ -79,10 +79,13 @@ public interface JobTag {} @Setter private boolean markedAsDeleted; /** - * When true, completed jobs return 303 See Other with redirect to result endpoint, rather than - * returning the result inline. This follows the SQL on FHIR unify-async specification. + * The asynchronous wire contract this job follows. Under {@link + * AsyncPattern#STANDARD_ASYNC_PATTERN} (the HL7 Asynchronous Interaction Request Pattern, spec) + * a completed job returns 303 See Other with a redirect to the result endpoint, rather than + * returning the result inline. Defaults to {@link AsyncPattern#BULK_DATA} and is never null. */ - @Setter private boolean redirectOnComplete; + @Setter private AsyncPattern pattern = AsyncPattern.BULK_DATA; /** * The last calculated progress percentage. When a job is at 100% that does not always indicate diff --git a/server/src/main/java/au/csiro/pathling/async/JobProvider.java b/server/src/main/java/au/csiro/pathling/async/JobProvider.java index b4d464d7ff..ca2d85656a 100644 --- a/server/src/main/java/au/csiro/pathling/async/JobProvider.java +++ b/server/src/main/java/au/csiro/pathling/async/JobProvider.java @@ -248,9 +248,11 @@ private static ResourceNotFoundException handleCancelledJob() { /** * Handles a completed job by returning its result or redirecting to the result endpoint. * - *

    If the job has {@code redirectOnComplete} enabled (following the SQL on FHIR unify-async - * specification), returns 303 See Other with a Location header pointing to the result endpoint. - * Otherwise, returns the result inline. + *

    If the job follows the {@link AsyncPattern#STANDARD_ASYNC_PATTERN} (the HL7 Asynchronous + * Interaction Request Pattern, spec), + * returns 303 See Other with a Location header pointing to the result endpoint. Otherwise, + * returns the result inline. * * @param job The completed job. * @param request The HTTP request for building the result URL. @@ -269,8 +271,9 @@ private IBaseResource handleCompletedJob( setAsyncCacheHeaders(response); } - // If redirect is enabled, return 303 See Other with Location header. - if (job.isRedirectOnComplete() && response != null) { + // Under the HL7 Asynchronous Interaction Request Pattern, return 303 See Other with a + // Location header pointing to the result endpoint. + if (job.getPattern() == AsyncPattern.STANDARD_ASYNC_PATTERN && response != null) { final String resultUrl = buildResultUrl(request, job.getId()); response.setStatus(HttpServletResponse.SC_SEE_OTHER); response.setHeader("Location", resultUrl); diff --git a/server/src/main/java/au/csiro/pathling/async/JobResultProvider.java b/server/src/main/java/au/csiro/pathling/async/JobResultProvider.java index e734c8b044..a53e692649 100644 --- a/server/src/main/java/au/csiro/pathling/async/JobResultProvider.java +++ b/server/src/main/java/au/csiro/pathling/async/JobResultProvider.java @@ -46,8 +46,9 @@ /** * Provides the $job-result operation for retrieving the result of a completed async job. This - * endpoint is used when operations are configured with {@code redirectOnComplete=true}, following - * the SQL on FHIR unify-async specification. + * endpoint is used by operations following the {@link AsyncPattern#STANDARD_ASYNC_PATTERN} (the HL7 + * Asynchronous Interaction Request Pattern, spec). * *

    The flow is: 1. Client polls $job endpoint until job completes 2. $job returns 303 See Other * with Location header pointing to $job-result 3. Client fetches result from $job-result endpoint diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java index a838f27dc7..6977379c50 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProvider.java @@ -17,6 +17,7 @@ package au.csiro.pathling.operations.sqlquery; +import au.csiro.pathling.async.AsyncPattern; import au.csiro.pathling.async.AsyncSupported; import au.csiro.pathling.async.PreAsyncValidation; import au.csiro.pathling.security.OperationAccess; @@ -84,7 +85,7 @@ public SqlQueryExportProvider( @SuppressWarnings({"unused", "java:S107"}) @Operation(name = "$sqlquery-export", idempotent = true) @OperationAccess("sqlquery-export") - @AsyncSupported(redirectOnComplete = true) + @AsyncSupported(pattern = AsyncPattern.STANDARD_ASYNC_PATTERN) @Nullable public Parameters export( @Nullable @OperationParam(name = "clientTrackingId") final String clientTrackingId, diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java index 7296256372..7019544d54 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryInstanceExportProvider.java @@ -17,6 +17,7 @@ package au.csiro.pathling.operations.sqlquery; +import au.csiro.pathling.async.AsyncPattern; import au.csiro.pathling.async.AsyncSupported; import au.csiro.pathling.async.PreAsyncValidation; import au.csiro.pathling.security.OperationAccess; @@ -107,7 +108,7 @@ public Class getResourceType() { @SuppressWarnings({"unused", "java:S107"}) @Operation(name = "$sqlquery-export", idempotent = true) @OperationAccess("sqlquery-export") - @AsyncSupported(redirectOnComplete = true) + @AsyncSupported(pattern = AsyncPattern.STANDARD_ASYNC_PATTERN) @Nullable public Parameters exportType( @Nullable @OperationParam(name = "clientTrackingId") final String clientTrackingId, @@ -138,7 +139,7 @@ public Parameters exportType( @SuppressWarnings({"unused", "java:S107"}) @Operation(name = "$sqlquery-export", idempotent = true) @OperationAccess("sqlquery-export") - @AsyncSupported(redirectOnComplete = true) + @AsyncSupported(pattern = AsyncPattern.STANDARD_ASYNC_PATTERN) @Nullable public Parameters exportInstance( @IdParam final IdType libraryId, diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java index 99cd5dc528..7d4cdcff00 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewDefinitionExportProvider.java @@ -20,6 +20,7 @@ import static au.csiro.pathling.security.SecurityAspect.getCurrentUserId; import au.csiro.pathling.async.AsyncJobContext; +import au.csiro.pathling.async.AsyncPattern; import au.csiro.pathling.async.AsyncSupported; import au.csiro.pathling.async.Job; import au.csiro.pathling.async.JobRegistry; @@ -160,7 +161,7 @@ public ViewDefinitionExportProvider( @SuppressWarnings({"unused", "java:S107"}) @Operation(name = "viewdefinition-export", idempotent = true) @OperationAccess("view-export") - @AsyncSupported(redirectOnComplete = true) + @AsyncSupported(pattern = AsyncPattern.STANDARD_ASYNC_PATTERN) @Nullable public Parameters export( @Nullable @OperationParam(name = "view.name") final List viewNames, diff --git a/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java b/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java index 214de5bd8a..9a79dd9108 100644 --- a/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java @@ -133,12 +133,12 @@ void jobResponseHasNoEtag() { @Test void completedJobWithRedirectReturns303SeeOther() { - // When redirectOnComplete is enabled, completed jobs should return 303 See Other with a - // Location header pointing to the result endpoint. + // Under the HL7 Asynchronous Interaction Request Pattern, completed jobs should return 303 See + // Other with a Location header pointing to the result endpoint. final CompletableFuture future = CompletableFuture.completedFuture(new Parameters()); final Job job = new Job<>(JOB_ID, "export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); final IBaseResource result = jobProvider.job(JOB_ID, request, response); @@ -152,8 +152,7 @@ void completedJobWithRedirectReturns303SeeOther() { @Test void completedJobWithoutRedirectReturns200WithResult() { - // When redirectOnComplete is not enabled (default), completed jobs return 200 OK with the - // inline result. + // Under the default BULK_DATA pattern, completed jobs return 200 OK with the inline result. final Parameters expectedResult = new Parameters(); expectedResult .addParameter() @@ -162,7 +161,7 @@ void completedJobWithoutRedirectReturns200WithResult() { final CompletableFuture future = CompletableFuture.completedFuture(expectedResult); final Job job = new Job<>(JOB_ID, "export", future, Optional.empty()); - // redirectOnComplete is false by default. + // The pattern defaults to BULK_DATA. jobRegistry.register(job); final IBaseResource result = jobProvider.job(JOB_ID, request, response); @@ -179,7 +178,7 @@ void inProgressJobReturns202RegardlessOfRedirectFlag() { // In-progress jobs always return 202 Accepted, regardless of the redirect setting. final CompletableFuture future = new CompletableFuture<>(); final Job job = new Job<>(JOB_ID, "export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobProvider.job(JOB_ID, request, response)) @@ -201,7 +200,7 @@ void completedJobWithRedirectIncludesServerBaseInLocation() { final CompletableFuture future = CompletableFuture.completedFuture(new Parameters()); final Job job = new Job<>(JOB_ID, "export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); jobProvider.job(JOB_ID, request, response); @@ -219,7 +218,7 @@ void completedJobWithRedirectSetsCacheHeaders() { final CompletableFuture future = CompletableFuture.completedFuture(new Parameters()); final Job job = new Job<>(JOB_ID, "export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); jobProvider.job(JOB_ID, request, response); diff --git a/server/src/test/java/au/csiro/pathling/async/JobResultProviderTest.java b/server/src/test/java/au/csiro/pathling/async/JobResultProviderTest.java index 3ed26f5045..c6814e3b01 100644 --- a/server/src/test/java/au/csiro/pathling/async/JobResultProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/async/JobResultProviderTest.java @@ -93,7 +93,7 @@ void successfulJobResultReturns200WithParameters() { final CompletableFuture future = CompletableFuture.completedFuture(expectedResult); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); final IBaseResource result = jobResultProvider.jobResult(JOB_ID, request, response); @@ -115,7 +115,7 @@ void failedJobResultReturnsOperationOutcome() { new InvalidRequestException("Test validation error"))); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); // The error should be converted and thrown. @@ -140,7 +140,7 @@ void inProgressJobResultReturns400BadRequest() { // Attempting to get the result of an in-progress job should return 400 Bad Request. final CompletableFuture future = new CompletableFuture<>(); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobResultProvider.jobResult(JOB_ID, request, response)) @@ -155,7 +155,7 @@ void resultSetsAppropriateHeaders() { final CompletableFuture future = CompletableFuture.completedFuture(expectedResult); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); // Set a response modification (like the Expires header from ViewDefinitionExportProvider). job.setResponseModification( httpServletResponse -> httpServletResponse.addHeader("Expires", "some-date")); @@ -188,7 +188,7 @@ void ownershipCheckEnforcedForResult() { // Job owned by "original-owner". final Job job = new Job<>(JOB_ID, "view-export", future, Optional.of("original-owner")); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobResultProvider.jobResult(JOB_ID, request, response)) @@ -217,7 +217,7 @@ void ownerMatchAllowsAccess() { CompletableFuture.completedFuture(expectedResult); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.of("job-owner-123")); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); final IBaseResource result = jobResultProvider.jobResult(JOB_ID, request, response); @@ -231,7 +231,7 @@ void cancelledJobReturns404() { final CompletableFuture future = new CompletableFuture<>(); future.cancel(false); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobResultProvider.jobResult(JOB_ID, request, response)) @@ -254,7 +254,7 @@ void interruptedJobThrowsInternalError() { } final Job job = new Job<>(JOB_ID, "view-export", mockFuture, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobResultProvider.jobResult(JOB_ID, request, response)) @@ -269,7 +269,7 @@ void errorUnwrappingHandlesDirectCause() { future.completeExceptionally(new InvalidRequestException("Direct error")); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobResultProvider.jobResult(JOB_ID, request, response)) @@ -287,7 +287,7 @@ void errorUnwrappingHandlesNestedCause() { "Outer wrapper", new InvalidRequestException("Nested error message"))); final Job job = new Job<>(JOB_ID, "view-export", future, Optional.empty()); - job.setRedirectOnComplete(true); + job.setPattern(AsyncPattern.STANDARD_ASYNC_PATTERN); jobRegistry.register(job); assertThatThrownBy(() -> jobResultProvider.jobResult(JOB_ID, request, response)) diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java index b6a79db993..990ea5f771 100644 --- a/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewDefinitionExportProviderIT.java @@ -529,7 +529,7 @@ private Map createSimplePatientViewDefinition() { } // ------------------------------------------------------------------------- - // 303 Redirect pattern tests (SQL on FHIR unify-async specification) + // 303 Redirect pattern tests (HL7 Asynchronous Interaction Request Pattern) // ------------------------------------------------------------------------- /** @@ -591,7 +591,7 @@ void exportAsyncWith303RedirectPattern() throws InterruptedException { Thread.sleep(500); } else if (status == HttpStatus.OK) { // Legacy behaviour without redirect - test passes but logs warning. - log.warn("Got 200 OK instead of 303. This indicates redirectOnComplete=false."); + log.warn("Got 200 OK instead of 303. This indicates the BULK_DATA pattern."); return; } else { throw new AssertionError("Unexpected status: " + status); From 0a42267c42acbbe838a45918586db91c7c6e59d9 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 08:51:56 +1000 Subject: [PATCH 012/162] test: Disable all export flags in the all-disabled assertion isAnyExportEnabled now also covers the SQL on FHIR asynchronous export operations, so the all-exports-disabled case must also disable the viewdefinition-export and sqlquery-export flags, both of which default to enabled. --- .../au/csiro/pathling/config/OperationConfigurationTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/au/csiro/pathling/config/OperationConfigurationTest.java b/server/src/test/java/au/csiro/pathling/config/OperationConfigurationTest.java index 8c97faba50..a703320e7b 100644 --- a/server/src/test/java/au/csiro/pathling/config/OperationConfigurationTest.java +++ b/server/src/test/java/au/csiro/pathling/config/OperationConfigurationTest.java @@ -86,11 +86,14 @@ void isAnyExportEnabledReturnsTrueWhenGroupExportEnabled() { @Test void isAnyExportEnabledReturnsFalseWhenAllExportDisabled() { - // Given: A configuration with all export operations disabled. + // Given: A configuration with all export operations disabled, including the SQL on FHIR + // asynchronous exports that also serve results through the $result endpoint. final OperationConfiguration config = new OperationConfiguration(); config.setExportEnabled(false); config.setPatientExportEnabled(false); config.setGroupExportEnabled(false); + config.setViewDefinitionExportEnabled(false); + config.setSqlQueryExportEnabled(false); // Then: isAnyExportEnabled should return false. assertThat(config.isAnyExportEnabled()).isFalse(); From 064c45b160de583be2022c14b1c33bd8859388a5 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 11:05:44 +1000 Subject: [PATCH 013/162] chore: Raise the server CI timeout to 45 minutes The SQL on FHIR export integration tests added to the server module push the full build-and-test suite past the previous 30-minute job limit, causing the run to be cancelled before completion. Raise the timeout on the test, pre-release, and release server workflows so the suite has room to finish. --- .github/workflows/server-pre-release.yml | 4 ++-- .github/workflows/server-release.yml | 4 ++-- .github/workflows/server-test.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/server-pre-release.yml b/.github/workflows/server-pre-release.yml index 0ac0fa97f5..adf3e1f4c9 100644 --- a/.github/workflows/server-pre-release.yml +++ b/.github/workflows/server-pre-release.yml @@ -17,7 +17,7 @@ jobs: pre-release-server: name: Build and push Docker image runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout code uses: actions/checkout@v4 @@ -73,7 +73,7 @@ jobs: -Dsonar.projectKey=aehrc_pathling_server -Dsonar.organization=aehrc \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.sarifReportPaths=../trivy-results-server.sarif - timeout-minutes: 30 + timeout-minutes: 45 - name: Save test reports if: always() diff --git a/.github/workflows/server-release.yml b/.github/workflows/server-release.yml index 209fa71e23..de713d4f8c 100644 --- a/.github/workflows/server-release.yml +++ b/.github/workflows/server-release.yml @@ -17,7 +17,7 @@ jobs: name: Build and push Docker image if: github.event_name == 'workflow_dispatch' || startsWith(github.ref_name, 'server-v') runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout code uses: actions/checkout@v4 @@ -65,7 +65,7 @@ jobs: -Dsonar.projectKey=aehrc_pathling_server -Dsonar.organization=aehrc \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.sarifReportPaths=../trivy-results-server.sarif - timeout-minutes: 30 + timeout-minutes: 45 - name: Save test reports if: always() diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 5ecc9ca929..0127052edd 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -22,7 +22,7 @@ jobs: test-server: name: Build and test server runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout code uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: -Dsonar.projectKey=aehrc_pathling_server -Dsonar.organization=aehrc \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.sarifReportPaths=trivy-results.sarif - timeout-minutes: 30 + timeout-minutes: 45 - name: Save test reports if: always() From 4cf636dc62efa3da6927ba034b90b5f202b27a58 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 11:39:20 +1000 Subject: [PATCH 014/162] test: Assert the operation list is populated before checking exclusion The disabled-operation assertion called doesNotContain on its own, which passes vacuously if the operation list is empty. Assert the still-enabled sqlquery-run operation is present first, so the test proves the list is populated and only sqlquery-export was dropped. --- .../java/au/csiro/pathling/fhir/ConformanceProviderTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java b/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java index fb87cfce74..f6c353c031 100644 --- a/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/fhir/ConformanceProviderTest.java @@ -289,7 +289,9 @@ void sqlQueryExportNotDeclaredWhenDisabled() { capabilityStatement.getRest().getFirst().getOperation().stream() .map(CapabilityStatementRestResourceOperationComponent::getName) .collect(Collectors.toSet()); - assertThat(operationNames).doesNotContain("sqlquery-export"); + // The sibling sqlquery-run operation remains enabled, so the operation list is still populated; + // only sqlquery-export should have been dropped. + assertThat(operationNames).contains("sqlquery-run").doesNotContain("sqlquery-export"); } @Test From 312a66c89c788c68765e652bb1f8ab93a2d25448 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 12:53:30 +1000 Subject: [PATCH 015/162] fix: Bump Netty to 4.1.135.Final to address transitive CVEs Resolves twelve Netty advisories reported against 4.1.133.Final, spanning HTTP/2 denial-of-service, TLS hostname verification bypass, and DNS resolver issues. All are fixed within the 4.1.x line at 4.1.135.Final, so no move to the 4.2.x series is required. The Netty BOM remains imported ahead of Spring Boot's so the pinned version continues to win. --- server/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pom.xml b/server/pom.xml index 6f54140b82..0489e14d03 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -43,7 +43,7 @@ 3.4.1 2.13 4.0.0 - 4.1.133.Final + 4.1.135.Final false false 2.18.6 From 450f7867f7707b3ffdee978dfcd285340f52f4a6 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 12:53:36 +1000 Subject: [PATCH 016/162] fix: Bump org.hl7.fhir.* core libraries to 6.9.10 Addresses CVE-2026-55471, an XML external entity vulnerability in XsltUtilities.saxonTransform via an unhardened Saxon TransformerFactory. HAPI FHIR 8.10.0 still bundles 6.9.4.1, so the existing override of org.hl7.fhir.r4 and org.hl7.fhir.utilities is raised from 6.9.7 to the patched 6.9.10. --- server/pom.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/pom.xml b/server/pom.xml index 0489e14d03..f014cdaede 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -411,19 +411,20 @@ 1.12.1 + to a HAPI release that bundles org.hl7.fhir.* >= 6.9.10 natively. --> ca.uhn.hapi.fhir org.hl7.fhir.r4 - 6.9.7 + 6.9.10 ca.uhn.hapi.fhir org.hl7.fhir.utilities - 6.9.7 + 6.9.10 + + /usr/bin/entrypoint.sh + + + src/main/jib + + + /usr/bin/entrypoint.sh + 755 + + + @@ -871,6 +882,28 @@ + + + org.codehaus.mojo + exec-maven-plugin + + + entrypoint-dispatch-test + test + + exec + + + ${skipTests} + bash + + ${project.basedir}/src/test/sh/entrypoint-test.sh + + + + + @@ -938,12 +971,23 @@ ${pathling.fhirServerDockerRepo}:${project.version} - - --add-exports=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/sun.util.calendar=ALL-UNNAMED - + + + /usr/bin/entrypoint.sh + + + src/main/jib + + + /usr/bin/entrypoint.sh + 755 + + + @@ -962,6 +1006,28 @@ + + + org.codehaus.mojo + exec-maven-plugin + + + entrypoint-dispatch-test + test + + exec + + + ${skipTests} + bash + + ${project.basedir}/src/test/sh/entrypoint-test.sh + + + + + From 8d7b721d15a3f6920de3d8a7ffdd5bc8923a73ef Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 22 Jun 2026 20:57:41 +1000 Subject: [PATCH 021/162] Correct Spark entrypoint reference to the runtime version The citation pointed to the Spark 4.0.0 entrypoint; the runtime is Spark 4.0.2, whose executor branch is identical. --- server/src/main/jib/usr/bin/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/jib/usr/bin/entrypoint.sh b/server/src/main/jib/usr/bin/entrypoint.sh index 485b252981..ee4ed77032 100755 --- a/server/src/main/jib/usr/bin/entrypoint.sh +++ b/server/src/main/jib/usr/bin/entrypoint.sh @@ -11,7 +11,7 @@ # command override, so the role selection must live in the image entrypoint. # # Modified from the Spark Kubernetes Docker image entrypoint script: -# https://github.com/apache/spark/blob/v4.0.0/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +# https://github.com/apache/spark/blob/v4.0.2/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh set -e From 57b1f521a5dffd5f0f5e073c62672e65422653b7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 25 Jun 2026 16:43:26 +1000 Subject: [PATCH 022/162] feat: Make cache backend first-byte timeout configurable The Varnish frontend cache previously inherited Varnish's default 60s first-byte timeout, which caused long-running synchronous Pathling queries to fail with a 503 before the server could respond. Expose the timeout as a PATHLING_FIRST_BYTE_TIMEOUT environment variable, substituted into the backend definition, and surface it through the Helm chart as pathlingCache.firstByteTimeout (defaulting to the previous 60s). --- deployment/cache/README.md | 4 +++ deployment/cache/chart/Chart.yaml | 2 +- deployment/cache/chart/README.md | 27 ++++++++++--------- .../cache/chart/templates/deployment.yaml | 2 ++ deployment/cache/chart/values.schema.json | 5 ++++ deployment/cache/chart/values.yaml | 5 ++++ deployment/cache/default.vcl | 4 +++ 7 files changed, 35 insertions(+), 14 deletions(-) diff --git a/deployment/cache/README.md b/deployment/cache/README.md index 7d4147b8c1..9ad82f7353 100644 --- a/deployment/cache/README.md +++ b/deployment/cache/README.md @@ -8,6 +8,10 @@ based upon the deployment: - `PATHLING_HOST`: The host name of the Pathling server. - `PATHLING_PORT`: The port number exposed by the Pathling server. +- `PATHLING_FIRST_BYTE_TIMEOUT`: The maximum time to wait for the first byte of + a backend response, expressed as a Varnish duration (e.g. `60s`, `10m`). Raise + this above the Varnish default of `60s` to accommodate long-running + synchronous Pathling queries. Copyright © 2065, Commonwealth Scientific and Industrial Research Organisation (CSIRO) ABN 41 687 119 230. Licensed under the Apache License, Version 2.0. diff --git a/deployment/cache/chart/Chart.yaml b/deployment/cache/chart/Chart.yaml index b03d0ce35b..2f6a27d371 100644 --- a/deployment/cache/chart/Chart.yaml +++ b/deployment/cache/chart/Chart.yaml @@ -8,7 +8,7 @@ name: pathling-cache description: A Varnish-based frontend cache optimised for use with Pathling Server icon: https://raw.githubusercontent.com/aehrc/pathling/main/media/logo-icon-colour-detail.svg type: application -version: 1.0.0 +version: 1.1.0 maintainers: - name: John Grimes email: John.Grimes@csiro.au diff --git a/deployment/cache/chart/README.md b/deployment/cache/chart/README.md index beb6191842..2a42a798a0 100644 --- a/deployment/cache/chart/README.md +++ b/deployment/cache/chart/README.md @@ -26,19 +26,20 @@ helm install my-cache ./chart \ ## Configuration -| Parameter | Description | Default | -| ------------------------------- | ------------------------------------------ | ------------------------------------- | -| `pathlingCache.image` | Container image to use | `ghcr.io/aehrc/pathling-cache:latest` | -| `pathlingCache.imagePullPolicy` | Image pull policy | `IfNotPresent` | -| `pathlingCache.replicas` | Number of replicas to deploy | `1` | -| `pathlingCache.pathlingHost` | Hostname of the Pathling server (required) | `~` | -| `pathlingCache.pathlingPort` | Port of the Pathling server (required) | `~` | -| `pathlingCache.service.type` | Kubernetes service type | `ClusterIP` | -| `pathlingCache.service.port` | Service port | `80` | -| `pathlingCache.resources` | CPU/memory resource requests and limits | `{}` | -| `pathlingCache.tolerations` | Pod tolerations | `[]` | -| `pathlingCache.affinity` | Pod affinity rules | `{}` | -| `pathlingCache.nodeSelector` | Node selector labels | `{}` | +| Parameter | Description | Default | +| -------------------------------- | --------------------------------------------------------- | ------------------------------------- | +| `pathlingCache.image` | Container image to use | `ghcr.io/aehrc/pathling-cache:latest` | +| `pathlingCache.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `pathlingCache.replicas` | Number of replicas to deploy | `1` | +| `pathlingCache.pathlingHost` | Hostname of the Pathling server (required) | `~` | +| `pathlingCache.pathlingPort` | Port of the Pathling server (required) | `~` | +| `pathlingCache.firstByteTimeout` | Backend first-byte timeout (Varnish duration, e.g. `10m`) | `60s` | +| `pathlingCache.service.type` | Kubernetes service type | `ClusterIP` | +| `pathlingCache.service.port` | Service port | `80` | +| `pathlingCache.resources` | CPU/memory resource requests and limits | `{}` | +| `pathlingCache.tolerations` | Pod tolerations | `[]` | +| `pathlingCache.affinity` | Pod affinity rules | `{}` | +| `pathlingCache.nodeSelector` | Node selector labels | `{}` | ## Examples diff --git a/deployment/cache/chart/templates/deployment.yaml b/deployment/cache/chart/templates/deployment.yaml index ed6afd4c50..79393aa2d7 100644 --- a/deployment/cache/chart/templates/deployment.yaml +++ b/deployment/cache/chart/templates/deployment.yaml @@ -32,6 +32,8 @@ spec: value: {{ required "pathlingCache.pathlingHost is required" .Values.pathlingCache.pathlingHost | quote }} - name: PATHLING_PORT value: {{ required "pathlingCache.pathlingPort is required" .Values.pathlingCache.pathlingPort | quote }} + - name: PATHLING_FIRST_BYTE_TIMEOUT + value: {{ .Values.pathlingCache.firstByteTimeout | quote }} livenessProbe: tcpSocket: port: http diff --git a/deployment/cache/chart/values.schema.json b/deployment/cache/chart/values.schema.json index 17ae6f7a34..75a5970282 100644 --- a/deployment/cache/chart/values.schema.json +++ b/deployment/cache/chart/values.schema.json @@ -33,6 +33,11 @@ "type": ["integer", "null"], "description": "Port of the Pathling server to cache (required)" }, + "firstByteTimeout": { + "type": "string", + "description": "Maximum time the cache waits for the first byte of a backend response, as a Varnish duration (e.g. '60s', '10m')", + "default": "60s" + }, "service": { "type": "object", "description": "Kubernetes service configuration", diff --git a/deployment/cache/chart/values.yaml b/deployment/cache/chart/values.yaml index 7681d8d0cb..55ae38ab1b 100644 --- a/deployment/cache/chart/values.yaml +++ b/deployment/cache/chart/values.yaml @@ -12,6 +12,11 @@ pathlingCache: pathlingHost: ~ pathlingPort: ~ + # Maximum time the cache waits for the first byte of a backend response, + # expressed as a Varnish duration (e.g. "60s", "10m"). Raise this when + # synchronous Pathling queries take longer than the Varnish default of 60s. + firstByteTimeout: "60s" + service: type: "ClusterIP" port: 80 diff --git a/deployment/cache/default.vcl b/deployment/cache/default.vcl index efe9e4380b..1d8db54556 100644 --- a/deployment/cache/default.vcl +++ b/deployment/cache/default.vcl @@ -7,6 +7,10 @@ import std; backend default { .host = "${PATHLING_HOST}"; .port = "${PATHLING_PORT}"; + // Maximum time to wait for the first byte of the backend response. Raised + // above the Varnish default of 60s to accommodate long-running synchronous + // Pathling queries. + .first_byte_timeout = ${PATHLING_FIRST_BYTE_TIMEOUT}; } sub vcl_recv { From 1a5aa37fd1b1ffd88ffaa01f46dabe07228c1182 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 25 Jun 2026 16:43:41 +1000 Subject: [PATCH 023/162] docs: Correct copyright year in cache README --- deployment/cache/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/cache/README.md b/deployment/cache/README.md index 9ad82f7353..acf685537c 100644 --- a/deployment/cache/README.md +++ b/deployment/cache/README.md @@ -13,5 +13,5 @@ based upon the deployment: this above the Varnish default of `60s` to accommodate long-running synchronous Pathling queries. -Copyright © 2065, Commonwealth Scientific and Industrial Research Organisation +Copyright © 2025, Commonwealth Scientific and Industrial Research Organisation (CSIRO) ABN 41 687 119 230. Licensed under the Apache License, Version 2.0. From 49390a62d3a0acb0411e00224bd14ac6e988d98c Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 26 Jun 2026 20:11:56 +1000 Subject: [PATCH 024/162] feat: Grant driver RBAC permissions to manage executor scratch PVCs Spark provisions on-demand persistent volume claims for executor local storage, so the driver service account needs permission to create and remove them. Without this the dynamically created scratch volumes fail to mount. --- deployment/helm/pathling/templates/role.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deployment/helm/pathling/templates/role.yaml b/deployment/helm/pathling/templates/role.yaml index e5072dc652..7f29a73ed3 100644 --- a/deployment/helm/pathling/templates/role.yaml +++ b/deployment/helm/pathling/templates/role.yaml @@ -6,6 +6,11 @@ rules: - apiGroups: [""] resources: ["pods", "services", "configmaps"] verbs: ["create", "get", "list", "watch", "delete"] + # Allow the driver to provision and remove the dynamically created scratch + # volumes used for executor local storage (claimName OnDemand). + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete"] - apiGroups: [""] resources: ["pods/log"] verbs: ["get", "list", "watch"] From d794187939d3dfee042d1d5491e39b4619b02134 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 08:47:12 +1000 Subject: [PATCH 025/162] feat: Add shared SQL on FHIR Library parser and dependency graph model Generalise the SQLQuery Library parser into a shared SqlLibraryParser that accepts both the SQLQuery and SQLView profiles, keyed by the Library.type code, requiring parameters be absent for a SQLView. Introduce the resolved dependency graph model (ResolvedDependency, ResolvedViewDefinition, ResolvedSqlView, ResolvedDependencyGraph) and a configurable maxDependencyDepth. Make the temp-view registration service key views by canonical identity rather than label so shared nodes materialise once and labels cannot collide across nodes. --- .../config/SqlQueryConfiguration.java | 10 ++ .../operations/sqlquery/ParsedSqlQuery.java | 23 ++- .../sqlquery/ResolvedDependency.java | 40 +++++ .../sqlquery/ResolvedDependencyGraph.java | 47 +++++ .../operations/sqlquery/ResolvedSqlView.java | 42 +++++ .../sqlquery/ResolvedViewDefinition.java | 41 +++++ ...braryParser.java => SqlLibraryParser.java} | 140 ++++++++++----- .../sqlquery/SqlQueryRequestParser.java | 4 +- .../operations/sqlquery/SqlValidator.java | 6 +- .../sqlquery/ViewRegistrationService.java | 111 ++++++++++-- .../config/SqlQueryConfigurationTest.java | 82 +++++++++ .../sqlquery/SqlLibraryFixtures.java | 168 ++++++++++++++++++ ...serTest.java => SqlLibraryParserTest.java} | 81 +++++++-- .../sqlquery/SqlQueryExportExecutorTest.java | 6 +- .../sqlquery/SqlQueryExportFormatIT.java | 4 +- .../sqlquery/SqlQueryExportProviderIT.java | 4 +- .../SqlQueryExportRequestParserTest.java | 6 +- .../SqlQueryExportTestConfiguration.java | 6 +- .../sqlquery/SqlQueryPipelineTest.java | 3 +- .../sqlquery/SqlQueryRequestParserTest.java | 2 +- .../sqlquery/SqlQueryRunDeltaIT.java | 6 +- .../sqlquery/SqlQueryRunProviderIT.java | 6 +- .../sqlquery/SqlQueryRunSecurityIT.java | 6 +- .../SqlQueryRunWithViewDefinitionsIT.java | 6 +- .../sqlquery/ViewRegistrationServiceTest.java | 43 +++++ 25 files changed, 792 insertions(+), 101 deletions(-) create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependency.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependencyGraph.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedSqlView.java create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedViewDefinition.java rename server/src/main/java/au/csiro/pathling/operations/sqlquery/{SqlQueryLibraryParser.java => SqlLibraryParser.java} (53%) create mode 100644 server/src/test/java/au/csiro/pathling/config/SqlQueryConfigurationTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java rename server/src/test/java/au/csiro/pathling/operations/sqlquery/{SqlQueryLibraryParserTest.java => SqlLibraryParserTest.java} (80%) diff --git a/server/src/main/java/au/csiro/pathling/config/SqlQueryConfiguration.java b/server/src/main/java/au/csiro/pathling/config/SqlQueryConfiguration.java index 9d0960bca4..94ebe1358e 100644 --- a/server/src/main/java/au/csiro/pathling/config/SqlQueryConfiguration.java +++ b/server/src/main/java/au/csiro/pathling/config/SqlQueryConfiguration.java @@ -49,4 +49,14 @@ public class SqlQueryConfiguration { */ @Min(1) private long timeoutSeconds = 60L; + + /** + * The maximum nesting depth of the dependency graph resolved for a single query. The top-level + * query's direct {@code relatedArtifact} dependencies sit at depth one; each further level of + * nested {@code SQLView} dependency increments the depth. A graph that nests deeper than this + * limit is rejected before any Spark work, guarding against accidental fan-out and runaway + * resolution. Real view graphs are shallow, so the default is generous while still bounded. + */ + @Min(1) + private int maxDependencyDepth = 10; } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java index 9af7985135..a60e0f8dc0 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java @@ -22,8 +22,8 @@ import lombok.Value; /** - * Represents a parsed SQLQuery Library resource containing the SQL text, ViewDefinition - * dependencies, and declared parameters. + * Represents a parsed SQL on FHIR Library resource ({@code SQLQuery} or {@code SQLView}) containing + * the SQL text, dependency references, declared parameters, and the resolved library type code. */ @Value public class ParsedSqlQuery { @@ -31,9 +31,24 @@ public class ParsedSqlQuery { /** The decoded SQL query text. */ @Nonnull String sql; - /** The ViewDefinition dependencies referenced in the SQL query. */ + /** The dependency references (to ViewDefinitions and/or SQLViews) referenced in the SQL. */ @Nonnull List viewReferences; - /** The declared parameters that can be bound at execution time. */ + /** The declared parameters that can be bound at execution time. Always empty for a SQLView. */ @Nonnull List declaredParameters; + + /** + * The SQL on FHIR library type code: {@link SqlLibraryParser#SQL_QUERY_TYPE_CODE} or {@link + * SqlLibraryParser#SQL_VIEW_TYPE_CODE}. + */ + @Nonnull String libraryTypeCode; + + /** + * Indicates whether this parsed query came from a {@code SQLView} Library. + * + * @return {@code true} if the library type is {@code sql-view} + */ + public boolean isView() { + return SqlLibraryParser.isView(libraryTypeCode); + } } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependency.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependency.java new file mode 100644 index 0000000000..5dcb94f9fa --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependency.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import jakarta.annotation.Nonnull; + +/** + * A resolved node in a SQL on FHIR dependency graph: either a {@link ResolvedViewDefinition} leaf + * or a {@link ResolvedSqlView}. Each node is identified by a stable canonical key that is the basis + * of its request-scoped temp-view name and of diamond deduplication. + * + * @author John Grimes + */ +public interface ResolvedDependency { + + /** + * Returns the stable canonical identity of the resolved resource. Two references to the same + * resource share a key, so a node is materialised only once per request, and the key cannot + * collide with a different resource even when both are reached under the same table label. + * + * @return the canonical key + */ + @Nonnull + String getCanonicalKey(); +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependencyGraph.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependencyGraph.java new file mode 100644 index 0000000000..b3de661d8a --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedDependencyGraph.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.Map; +import lombok.Value; + +/** + * The fully resolved dependency graph for a single query: the transitive set of {@link + * ResolvedDependency} nodes reachable from the top-level query, together with the mapping from the + * top-level query's own table labels to the nodes they reference. Produced during request + * preparation (no Spark) and materialised, bottom-up, at execution. + * + * @author John Grimes + */ +@Value +public class ResolvedDependencyGraph { + + /** + * The nodes in topological order: every node appears after all of its dependencies, so + * materialising the list in order guarantees each node's children already exist as temp views. + */ + @Nonnull List orderedNodes; + + /** The top-level query's local table label to the canonical key of the node it references. */ + @Nonnull Map topLevelKeysByLabel; + + /** Lookup of every node by its canonical key, for deduplication and materialisation. */ + @Nonnull Map nodesByKey; +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedSqlView.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedSqlView.java new file mode 100644 index 0000000000..f9ac5bc5ce --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedSqlView.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import jakarta.annotation.Nonnull; +import java.util.Map; +import lombok.Value; + +/** + * A resolved {@code SQLView} node. Carries the view's SQL and the mapping from each table label the + * SQL uses to the canonical key of the child node that label resolves to, so the SQL can be + * rewritten against the children's request-scoped temp views at materialisation time. + * + * @author John Grimes + */ +@Value +public class ResolvedSqlView implements ResolvedDependency { + + /** The stable canonical identity of the SQLView Library. */ + @Nonnull String canonicalKey; + + /** The view's SQL text. */ + @Nonnull String sql; + + /** This view's local table label to the canonical key of the resolved child it references. */ + @Nonnull Map childKeysByLabel; +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedViewDefinition.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedViewDefinition.java new file mode 100644 index 0000000000..bb019e2542 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ResolvedViewDefinition.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.views.FhirView; +import jakarta.annotation.Nonnull; +import lombok.Value; + +/** + * A resolved leaf node wrapping a parsed {@link FhirView}. A {@code ViewDefinition} projects a + * single FHIR resource type and never declares further dependencies, so it is always a leaf of the + * dependency graph. + * + * @author John Grimes + */ +@Value +public class ResolvedViewDefinition implements ResolvedDependency { + + /** The stable canonical identity of the ViewDefinition. */ + @Nonnull String canonicalKey; + + /** + * The parsed view, ready for execution. Its resource drives the projected-resource READ check. + */ + @Nonnull FhirView view; +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryLibraryParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java similarity index 53% rename from server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryLibraryParser.java rename to server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java index 28bdea09e4..6dd8fd4629 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryLibraryParser.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java @@ -19,6 +19,7 @@ import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -34,99 +35,123 @@ import org.springframework.stereotype.Component; /** - * Parses a FHIR R4 Library resource conforming to the SQLQuery profile. Extracts the SQL text, - * ViewDefinition dependencies, and parameter declarations, and enforces the profile invariants. + * Parses a FHIR R4 Library resource conforming to the SQL on FHIR {@code SQLQuery} or {@code + * SQLView} profile. Extracts the SQL text, dependency references, and (for {@code SQLQuery}) + * parameter declarations, and enforces the profile invariants common to both. * - *

    The SQLQuery profile requires: + *

    The two profiles are near-twins. Both require: * *

      - *
    • {@code Library.type} carrying a coding of {@code sql-query} from the SQL on FHIR library - * types code system. + *
    • {@code Library.type} carrying a coding from the SQL on FHIR library types code system - + * {@code sql-query} for a {@code SQLQuery}, {@code sql-view} for a {@code SQLView}. *
    • A {@code content} entry with content type starting with {@code application/sql} containing * Base64-encoded SQL text. *
    • Each {@code relatedArtifact} of type {@code depends-on}, with a label matching {@code - * ^[A-Za-z][A-Za-z0-9_]*$} and a {@code resource} canonical URL pointing at the referenced - * ViewDefinition. - *
    • Each {@code parameter} declared with {@code use = in} and a name and type. + * ^[A-Za-z][A-Za-z0-9_]*$} and a {@code resource} reference pointing at the referenced {@code + * ViewDefinition} or {@code SQLView}. *
    * + *

    They differ in only one rule: a {@code SQLQuery} may declare input {@code parameter}s (each + * {@code use = in}), whereas a {@code SQLView} SHALL NOT declare any parameter. + * + * @author John Grimes * @see SQLQuery + * @see SQLView */ @Component -public class SqlQueryLibraryParser { +public class SqlLibraryParser { private static final String SQL_CONTENT_TYPE_PREFIX = "application/sql"; - /** Code system identifying the SQLQuery Library profile. */ + /** Code system identifying the SQL on FHIR Library profiles. */ public static final String LIBRARY_TYPE_SYSTEM = "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes"; - /** {@link #LIBRARY_TYPE_SYSTEM} code identifying a SQLQuery Library. */ - public static final String LIBRARY_TYPE_CODE = "sql-query"; + /** {@link #LIBRARY_TYPE_SYSTEM} code identifying a {@code SQLQuery} Library. */ + public static final String SQL_QUERY_TYPE_CODE = "sql-query"; + + /** {@link #LIBRARY_TYPE_SYSTEM} code identifying a {@code SQLView} Library. */ + public static final String SQL_VIEW_TYPE_CODE = "sql-view"; private static final Pattern LABEL_PATTERN = Pattern.compile("^[A-Za-z]\\w*$"); /** - * Parses a Library resource into a {@link ParsedSqlQuery}. + * Parses a Library resource into a {@link ParsedSqlQuery}, accepting either a {@code SQLQuery} or + * a {@code SQLView}. * * @param library the Library resource to parse - * @return the parsed SQL query containing SQL text, view references, and parameter declarations - * @throws InvalidRequestException if the Library does not conform to the SQLQuery profile + * @return the parsed query carrying SQL text, dependency references, parameter declarations, and + * the resolved library type code + * @throws InvalidRequestException if the Library does not conform to either profile */ @Nonnull public ParsedSqlQuery parse(@Nonnull final Library library) { - validateLibraryType(library); - final String sql = extractSql(library); + final String typeCode = resolveLibraryTypeCode(library); + final boolean isView = SQL_VIEW_TYPE_CODE.equals(typeCode); + final String sql = extractSql(library, typeCode); final List viewReferences = extractViewReferences(library); - final List parameters = extractParameters(library); - return new ParsedSqlQuery(sql, viewReferences, parameters); + final List parameters = extractParameters(library, isView); + return new ParsedSqlQuery(sql, viewReferences, parameters, typeCode); } /** - * Verifies that the Library carries the SQLQuery profile's type coding. The check accepts any - * coding with the expected system and code, regardless of additional codings, so that authors can - * layer their own classifications without breaking conformance. + * Resolves the SQL on FHIR library type code carried by {@code Library.type}. The check accepts a + * coding with the expected system and either the {@code sql-query} or {@code sql-view} code, + * regardless of additional codings, so that authors can layer their own classifications without + * breaking conformance. + * + * @return the matched type code ({@code sql-query} or {@code sql-view}) + * @throws InvalidRequestException if no recognised SQL on FHIR coding is present */ - private void validateLibraryType(@Nonnull final Library library) { + @Nonnull + private String resolveLibraryTypeCode(@Nonnull final Library library) { final CodeableConcept type = library.getType(); if (type == null || type.isEmpty()) { throw new InvalidRequestException( - "SQLQuery Library must declare Library.type with the SQLQuery coding (" + "SQL on FHIR Library must declare Library.type with a coding from " + LIBRARY_TYPE_SYSTEM - + "#" - + LIBRARY_TYPE_CODE + + " (" + + SQL_QUERY_TYPE_CODE + + " or " + + SQL_VIEW_TYPE_CODE + ")"); } for (final Coding coding : type.getCoding()) { - if (LIBRARY_TYPE_SYSTEM.equals(coding.getSystem()) - && LIBRARY_TYPE_CODE.equals(coding.getCode())) { - return; + if (LIBRARY_TYPE_SYSTEM.equals(coding.getSystem())) { + final String code = coding.getCode(); + if (SQL_QUERY_TYPE_CODE.equals(code) || SQL_VIEW_TYPE_CODE.equals(code)) { + return code; + } } } throw new InvalidRequestException( - "SQLQuery Library.type must include a coding with system " + "SQL on FHIR Library.type must include a coding with system " + LIBRARY_TYPE_SYSTEM + " and code " - + LIBRARY_TYPE_CODE); + + SQL_QUERY_TYPE_CODE + + " or " + + SQL_VIEW_TYPE_CODE); } /** * Extracts the SQL text from the Library's content entries. * * @param library the Library resource + * @param typeCode the resolved library type code, used in the error message * @return the decoded SQL text * @throws InvalidRequestException if no SQL content is found or the content is invalid */ @Nonnull - private String extractSql(@Nonnull final Library library) { + private String extractSql(@Nonnull final Library library, @Nonnull final String typeCode) { for (final Attachment attachment : library.getContent()) { final String contentType = attachment.getContentType(); if (contentType != null && contentType.startsWith(SQL_CONTENT_TYPE_PREFIX)) { final byte[] data = attachment.getData(); if (data == null || data.length == 0) { throw new InvalidRequestException( - "SQLQuery Library has an application/sql content entry with no data"); + "SQL on FHIR Library has an application/sql content entry with no data"); } // The data is Base64-encoded in the FHIR resource. HAPI decodes it automatically when // using getData(), so we can use it directly. @@ -134,15 +159,17 @@ private String extractSql(@Nonnull final Library library) { } } throw new InvalidRequestException( - "SQLQuery Library must contain a content entry with content type application/sql"); + "A " + + typeCode + + " Library must contain a content entry with content type application/sql"); } /** - * Extracts ViewDefinition references from the Library's related artifacts, enforcing that each - * artifact is of type {@code depends-on} with a label matching the SQLQuery profile pattern. + * Extracts dependency references from the Library's related artifacts, enforcing that each + * artifact is of type {@code depends-on} with a label matching the SQL on FHIR profile pattern. * * @param library the Library resource - * @return the list of view artifact references + * @return the list of dependency references */ @Nonnull private List extractViewReferences(@Nonnull final Library library) { @@ -150,7 +177,7 @@ private List extractViewReferences(@Nonnull final Library for (final RelatedArtifact artifact : library.getRelatedArtifact()) { if (artifact.getType() != RelatedArtifactType.DEPENDSON) { throw new InvalidRequestException( - "SQLQuery Library relatedArtifact must have type 'depends-on', but found '" + "SQL on FHIR Library relatedArtifact must have type 'depends-on', but found '" + (artifact.getType() == null ? "null" : artifact.getType().toCode()) + "'"); } @@ -158,18 +185,18 @@ private List extractViewReferences(@Nonnull final Library final String resource = artifact.getResource(); if (label == null || label.isBlank()) { throw new InvalidRequestException( - "Each relatedArtifact in the SQLQuery Library must have a label"); + "Each relatedArtifact in the SQL on FHIR Library must have a label"); } if (!LABEL_PATTERN.matcher(label).matches()) { throw new InvalidRequestException( - "SQLQuery Library relatedArtifact label '" + "SQL on FHIR Library relatedArtifact label '" + label + "' does not match the required pattern " + LABEL_PATTERN.pattern()); } if (resource == null || resource.isBlank()) { throw new InvalidRequestException( - "Each relatedArtifact in the SQLQuery Library must have a resource reference"); + "Each relatedArtifact in the SQL on FHIR Library must have a resource reference"); } references.add(new ViewArtifactReference(label, resource)); } @@ -177,14 +204,27 @@ private List extractViewReferences(@Nonnull final Library } /** - * Extracts parameter declarations from the Library's parameter entries, enforcing that each - * declaration is an input ({@code use = in}) and carries both a name and a type. + * Extracts parameter declarations from the Library's parameter entries. A {@code SQLView} SHALL + * NOT declare any parameter and is rejected if it does. For a {@code SQLQuery}, each declaration + * must be an input ({@code use = in}) and carry both a name and a type. * * @param library the Library resource - * @return the list of parameter declarations + * @param isView whether the Library is a {@code SQLView} + * @return the list of parameter declarations (always empty for a {@code SQLView}) + * @throws InvalidRequestException if a {@code SQLView} declares any parameter, or a {@code + * SQLQuery} parameter is malformed */ @Nonnull - private List extractParameters(@Nonnull final Library library) { + private List extractParameters( + @Nonnull final Library library, final boolean isView) { + if (isView) { + if (!library.getParameter().isEmpty()) { + throw new InvalidRequestException( + "A " + SQL_VIEW_TYPE_CODE + " Library must not declare any parameter"); + } + return List.of(); + } + final List parameters = new ArrayList<>(); for (final ParameterDefinition param : library.getParameter()) { final String name = param.getName(); @@ -209,4 +249,14 @@ private List extractParameters(@Nonnull final Library l } return parameters; } + + /** + * Indicates whether the given library type code denotes a {@code SQLView}. + * + * @param typeCode the SQL on FHIR library type code + * @return {@code true} if the code is {@code sql-view} + */ + public static boolean isView(@Nullable final String typeCode) { + return SQL_VIEW_TYPE_CODE.equals(typeCode); + } } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java index 202f2856bc..a9e5655548 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParser.java @@ -61,7 +61,7 @@ public class SqlQueryRequestParser { private static final Set INTEGER_LIKE_TYPES = Set.of("integer", "unsignedInt", "positiveInt"); - @Nonnull private final SqlQueryLibraryParser libraryParser; + @Nonnull private final SqlLibraryParser libraryParser; /** * Constructs a new SqlQueryRequestParser. @@ -69,7 +69,7 @@ public class SqlQueryRequestParser { * @param libraryParser parser for the SQLQuery Library profile */ @Autowired - public SqlQueryRequestParser(@Nonnull final SqlQueryLibraryParser libraryParser) { + public SqlQueryRequestParser(@Nonnull final SqlLibraryParser libraryParser) { this.libraryParser = libraryParser; } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java index a6b008fceb..6906e4cfa3 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlValidator.java @@ -317,9 +317,9 @@ public SqlValidator(@Nonnull final SparkSession sparkSession) { } /** - * Pattern that legitimate relation identifiers must match. Mirrors the {@code - * SqlQueryLibraryParser} label pattern so that anything Spark's parser produces as an - * UnresolvedRelation but which could not have been a declared label is rejected outright. + * Pattern that legitimate relation identifiers must match. Mirrors the {@code SqlLibraryParser} + * label pattern so that anything Spark's parser produces as an UnresolvedRelation but which could + * not have been a declared label is rejected outright. */ private static final Pattern LABEL_PATTERN = Pattern.compile("^[A-Za-z]\\w*$"); diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java index b6d1761de9..d067b4dfa7 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java @@ -118,20 +118,86 @@ public Map registerViews( } /** - * Registers an already-built dataset under a request-scoped temp view name and returns that name. - * Visible for tests that need to exercise the temp-view namespacing without going through - * FhirViewExecutor. + * Registers an already-built dataset under a request-scoped temp view name derived from the given + * identifier (a dependency's canonical key, or a bare label in the single-level tests) and + * returns that name. + * + * @param identifier the canonical key (or label) the temp view materialises + * @param dataset the dataset to register + * @param requestId the per-request id used to namespace the registered view name + * @return the registered temp view name */ @Nonnull String registerDataset( - @Nonnull final String label, + @Nonnull final String identifier, @Nonnull final Dataset dataset, @Nonnull final String requestId) { - final String tempViewName = resolveTempViewName(requestId, label); + final String tempViewName = resolveTempViewName(requestId, identifier); dataset.createOrReplaceTempView(tempViewName); return tempViewName; } + /** + * Builds the dataset for a resolved {@code ViewDefinition} leaf by executing its parsed view + * against the data source. The result is not registered; the caller registers it once any + * required validation has passed. + * + * @param view the parsed view to execute + * @param dataSource the data source backing view execution + * @return the view's result dataset + * @throws InvalidRequestException if the view cannot be executed + */ + @Nonnull + public Dataset buildViewDefinition( + @Nonnull final FhirView view, @Nonnull final DataSource dataSource) { + final FhirViewExecutor executor = + new FhirViewExecutor(fhirContext, dataSource, queryConfiguration); + try { + return executor.buildQuery(view); + } catch (final Exception e) { + throw new InvalidRequestException( + "Failed to execute ViewDefinition for resource type '" + + view.getResource() + + "': " + + e.getMessage()); + } + } + + /** + * Builds the dataset for a resolved {@code SQLView} node by rewriting its SQL so each of its + * table labels points at the already-registered temp view of the child it resolves to, then + * running that SQL. The result is not registered; the caller validates the analysed plan and then + * registers it. + * + * @param node the resolved SQLView node + * @param registeredByKey the temp view names of already-materialised nodes, keyed by canonical + * key + * @return the SQLView's result dataset + */ + @Nonnull + public Dataset buildSqlView( + @Nonnull final ResolvedSqlView node, @Nonnull final Map registeredByKey) { + final Map labelToViewName = new LinkedHashMap<>(); + node.getChildKeysByLabel() + .forEach( + (label, childKey) -> { + final String viewName = registeredByKey.get(childKey); + if (viewName == null) { + // Topological ordering guarantees children are materialised first; a miss here is + // an internal invariant violation, not a client error. + throw new IllegalStateException( + "Child dependency '" + + childKey + + "' for label '" + + label + + "' was not materialised before the SQLView that references it"); + } + labelToViewName.put(label, viewName); + }); + final String rewrittenSql = rewriteSql(node.getSql(), labelToViewName); + return sparkSession.sql(rewrittenSql); + } + /** * Drops the specified temporary views from the Spark session. * @@ -324,15 +390,22 @@ private static int copyOrRewriteIdentifier( } /** - * Constructs the request-scoped temp view name for a given label. + * Constructs the request-scoped temp view name for a node identifier. The identifier is either a + * dependency's canonical key (the production case, so a shared node materialises once and labels + * cannot collide across nodes) or, for the existing single-level tests, a bare table label. * - *

    The request id is sanitised to keep the resulting identifier valid for use as a Spark temp - * view name (HAPI default is 16 alphanumerics, but {@code X-Request-ID} can carry arbitrary - * characters). + *

    Both the request id and the identifier are sanitised so that the resulting Spark temp view + * name is a legal identifier (HAPI request ids are alphanumeric, but {@code X-Request-ID} and + * canonical keys such as {@code ViewDefinition/patient-view} can carry slashes and dashes). + * + * @param requestId the per-request id used to namespace registered view names + * @param identifier the canonical key (or label) the temp view materialises + * @return the temp view name */ @Nonnull - static String resolveTempViewName(@Nonnull final String requestId, @Nonnull final String label) { - return VIEW_NAME_PREFIX + sanitiseRequestId(requestId) + "_" + label; + static String resolveTempViewName( + @Nonnull final String requestId, @Nonnull final String identifier) { + return VIEW_NAME_PREFIX + sanitiseRequestId(requestId) + "_" + sanitiseIdentifier(identifier); } /** @@ -349,4 +422,20 @@ private static String sanitiseRequestId(@Nonnull final String requestId) { } return "r" + Integer.toUnsignedString(requestId.hashCode(), 16); } + + /** + * Renders a node identifier as a Spark-safe temp view name segment. A bare word identifier (a + * validated table label) is used verbatim, preserving the single-level naming. Any identifier + * containing characters illegal in a Spark identifier (a canonical key such as {@code + * ViewDefinition/patient-view}) has those characters replaced with underscores and a hash of the + * original appended, so that two distinct keys never collapse to the same name. + */ + @Nonnull + private static String sanitiseIdentifier(@Nonnull final String identifier) { + if (!UNSAFE_REQUEST_ID_CHARS.matcher(identifier).find()) { + return identifier; + } + final String cleaned = UNSAFE_REQUEST_ID_CHARS.matcher(identifier).replaceAll("_"); + return cleaned + "_" + Integer.toUnsignedString(identifier.hashCode(), 16); + } } diff --git a/server/src/test/java/au/csiro/pathling/config/SqlQueryConfigurationTest.java b/server/src/test/java/au/csiro/pathling/config/SqlQueryConfigurationTest.java new file mode 100644 index 0000000000..b41a26016a --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/config/SqlQueryConfigurationTest.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link SqlQueryConfiguration}, covering the {@code maxDependencyDepth} default and + * its {@code @Min(1)} validation. + * + * @author John Grimes + */ +class SqlQueryConfigurationTest { + + private Validator validator; + + @BeforeEach + void setUp() { + validator = Validation.buildDefaultValidatorFactory().getValidator(); + } + + @Test + void defaultMaxDependencyDepthIsTen() { + // The default must be a generous-but-bounded value so that real, shallow view graphs are never + // rejected while pathological fan-out is still capped. + assertThat(new SqlQueryConfiguration().getMaxDependencyDepth()).isEqualTo(10); + } + + @Test + void acceptsPositiveMaxDependencyDepth() { + final SqlQueryConfiguration config = new SqlQueryConfiguration(); + config.setMaxDependencyDepth(1); + + final Set> violations = validator.validate(config); + + assertThat(violations).isEmpty(); + } + + @Test + void rejectsZeroMaxDependencyDepth() { + // A depth of zero would forbid even a single dependency, which is nonsensical for a feature + // whose purpose is dependency resolution. + final SqlQueryConfiguration config = new SqlQueryConfiguration(); + config.setMaxDependencyDepth(0); + + final Set> violations = validator.validate(config); + + assertThat(violations).isNotEmpty(); + } + + @Test + void rejectsNegativeMaxDependencyDepth() { + final SqlQueryConfiguration config = new SqlQueryConfiguration(); + config.setMaxDependencyDepth(-5); + + final Set> violations = validator.validate(config); + + assertThat(violations).isNotEmpty(); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java new file mode 100644 index 0000000000..f9db0ed393 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java @@ -0,0 +1,168 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_VIEW_TYPE_CODE; + +import jakarta.annotation.Nonnull; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import org.hl7.fhir.r4.model.Attachment; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.ParameterDefinition; +import org.hl7.fhir.r4.model.ParameterDefinition.ParameterUse; +import org.hl7.fhir.r4.model.RelatedArtifact; +import org.hl7.fhir.r4.model.RelatedArtifact.RelatedArtifactType; + +/** + * Test fixtures for constructing {@code SQLQuery} and {@code SQLView} {@link Library} resources. + * The builders mirror the SQL on FHIR profiles: a {@code Library.type} coding, one {@code + * application/sql} content entry carrying the Base64-encoded SQL, {@code depends-on} {@code + * relatedArtifact} dependencies (label plus resource reference), and, for {@code SQLQuery}, + * optional input parameters. + * + * @author John Grimes + */ +public final class SqlLibraryFixtures { + + private SqlLibraryFixtures() { + // Utility class. + } + + /** + * Builds a {@code SQLQuery} Library carrying the given SQL, with no dependencies or parameters. + * + * @param sql the SQL text to embed as Base64 {@code application/sql} content + * @return a minimal {@code SQLQuery} Library + */ + @Nonnull + public static Library sqlQuery(@Nonnull final String sql) { + return baseLibrary(SQL_QUERY_TYPE_CODE, sql); + } + + /** + * Builds a {@code SQLQuery} Library carrying the given SQL and a single {@code depends-on} + * dependency. + * + * @param sql the SQL text to embed + * @param label the dependency table label + * @param resource the dependency resource reference + * @return a {@code SQLQuery} Library with one dependency + */ + @Nonnull + public static Library sqlQuery( + @Nonnull final String sql, @Nonnull final String label, @Nonnull final String resource) { + final Library library = baseLibrary(SQL_QUERY_TYPE_CODE, sql); + addDependency(library, label, resource); + return library; + } + + /** + * Builds a {@code SQLView} Library carrying the given SQL, with no dependencies or parameters. + * + * @param sql the SQL text to embed as Base64 {@code application/sql} content + * @return a minimal {@code SQLView} Library + */ + @Nonnull + public static Library sqlView(@Nonnull final String sql) { + return baseLibrary(SQL_VIEW_TYPE_CODE, sql); + } + + /** + * Builds a {@code SQLView} Library carrying the given SQL and a single {@code depends-on} + * dependency. + * + * @param sql the SQL text to embed + * @param label the dependency table label + * @param resource the dependency resource reference + * @return a {@code SQLView} Library with one dependency + */ + @Nonnull + public static Library sqlView( + @Nonnull final String sql, @Nonnull final String label, @Nonnull final String resource) { + final Library library = baseLibrary(SQL_VIEW_TYPE_CODE, sql); + addDependency(library, label, resource); + return library; + } + + /** + * Builds a {@code SQLView} Library carrying the given SQL and a set of {@code depends-on} + * dependencies, preserving the iteration order of the supplied map. + * + * @param sql the SQL text to embed + * @param dependenciesByLabel the dependencies keyed by table label, each value a resource + * reference + * @return a {@code SQLView} Library with the given dependencies + */ + @Nonnull + public static Library sqlView( + @Nonnull final String sql, @Nonnull final Map dependenciesByLabel) { + final Library library = baseLibrary(SQL_VIEW_TYPE_CODE, sql); + dependenciesByLabel.forEach((label, resource) -> addDependency(library, label, resource)); + return library; + } + + /** + * Adds a {@code depends-on} {@code relatedArtifact} to the given Library. + * + * @param library the Library to extend + * @param label the dependency table label + * @param resource the dependency resource reference + */ + public static void addDependency( + @Nonnull final Library library, @Nonnull final String label, @Nonnull final String resource) { + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel(label) + .setResource(resource)); + } + + /** + * Adds an input parameter declaration to the given Library. + * + * @param library the Library to extend + * @param name the parameter name + * @param type the FHIR primitive type code + */ + public static void addParameter( + @Nonnull final Library library, @Nonnull final String name, @Nonnull final String type) { + library.addParameter( + new ParameterDefinition().setName(name).setType(type).setUse(ParameterUse.IN)); + } + + /** Builds a Library with the given SQL on FHIR library-type code and embedded SQL content. */ + @Nonnull + private static Library baseLibrary(@Nonnull final String typeCode, @Nonnull final String sql) { + final Library library = new Library(); + library.setStatus(PublicationStatus.ACTIVE); + library.setType( + new CodeableConcept() + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(typeCode))); + final Attachment content = new Attachment(); + content.setContentType("application/sql"); + content.setData(sql.getBytes(StandardCharsets.UTF_8)); + library.addContent(content); + return library; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryLibraryParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java similarity index 80% rename from server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryLibraryParserTest.java rename to server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java index 4f0a1b434e..0528930ef1 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryLibraryParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java @@ -17,6 +17,9 @@ package au.csiro.pathling.operations.sqlquery; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_VIEW_TYPE_CODE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -31,17 +34,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -/** Unit tests for {@link SqlQueryLibraryParser}. */ -class SqlQueryLibraryParserTest { +/** Unit tests for {@link SqlLibraryParser} covering both the SQLQuery and SQLView profiles. */ +class SqlLibraryParserTest { - private static final String LIBRARY_TYPE_SYSTEM = - "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes"; - - private SqlQueryLibraryParser parser; + private SqlLibraryParser parser; @BeforeEach void setUp() { - parser = new SqlQueryLibraryParser(); + parser = new SqlLibraryParser(); } @Test @@ -98,6 +98,61 @@ void handlesEmptyViewReferencesAndParameters() { assertThat(result.getDeclaredParameters()).isEmpty(); } + @Test + void reportsSqlQueryAsNotAView() { + final ParsedSqlQuery result = parser.parse(createMinimalLibrary("SELECT 1")); + assertThat(result.getLibraryTypeCode()).isEqualTo(SQL_QUERY_TYPE_CODE); + assertThat(result.isView()).isFalse(); + } + + // --------------------------------------------------------------------------- + // SQLView profile. + // --------------------------------------------------------------------------- + + @Test + void parsesSqlViewWithTypeCodeSqlAndDependencies() { + final Library library = SqlLibraryFixtures.sqlView("SELECT * FROM patient_view"); + SqlLibraryFixtures.addDependency(library, "patient_view", "ViewDefinition/patient-view"); + SqlLibraryFixtures.addDependency(library, "base", "Library/base"); + + final ParsedSqlQuery result = parser.parse(library); + + assertThat(result.getLibraryTypeCode()).isEqualTo(SQL_VIEW_TYPE_CODE); + assertThat(result.isView()).isTrue(); + assertThat(result.getSql()).isEqualTo("SELECT * FROM patient_view"); + assertThat(result.getViewReferences()).hasSize(2); + assertThat(result.getViewReferences().get(0).getLabel()).isEqualTo("patient_view"); + assertThat(result.getViewReferences().get(0).getCanonicalUrl()) + .isEqualTo("ViewDefinition/patient-view"); + assertThat(result.getViewReferences().get(1).getCanonicalUrl()).isEqualTo("Library/base"); + assertThat(result.getDeclaredParameters()).isEmpty(); + } + + @Test + void rejectsSqlViewDeclaringAParameter() { + // A SQLView SHALL NOT declare parameters; doing so is a 400. + final Library library = SqlLibraryFixtures.sqlView("SELECT 1"); + SqlLibraryFixtures.addParameter(library, "min_age", "integer"); + + assertThatThrownBy(() -> parser.parse(library)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining(SQL_VIEW_TYPE_CODE) + .hasMessageContaining("parameter"); + } + + @Test + void stillParsesSqlQueryWithParameters() { + // The shared parser must continue to accept SQLQuery parameters unchanged. + final Library library = SqlLibraryFixtures.sqlQuery("SELECT * FROM t WHERE age > :min_age"); + SqlLibraryFixtures.addParameter(library, "min_age", "integer"); + + final ParsedSqlQuery result = parser.parse(library); + + assertThat(result.isView()).isFalse(); + assertThat(result.getDeclaredParameters()).hasSize(1); + assertThat(result.getDeclaredParameters().get(0).getName()).isEqualTo("min_age"); + } + // --------------------------------------------------------------------------- // SQL content rejections. // --------------------------------------------------------------------------- @@ -156,7 +211,7 @@ void rejectsLibraryWithoutTypeCoding() { assertThatThrownBy(() -> parser.parse(library)) .isInstanceOf(InvalidRequestException.class) .hasMessageContaining("Library.type") - .hasMessageContaining("sql-query"); + .hasMessageContaining(SQL_QUERY_TYPE_CODE); } @Test @@ -177,7 +232,7 @@ void rejectsLibraryWithUnrelatedTypeCoding() { assertThatThrownBy(() -> parser.parse(library)) .isInstanceOf(InvalidRequestException.class) .hasMessageContaining(LIBRARY_TYPE_SYSTEM) - .hasMessageContaining("sql-query"); + .hasMessageContaining(SQL_QUERY_TYPE_CODE); } @Test @@ -193,7 +248,7 @@ void acceptsLibraryWithAdditionalTypeCodings() { .addCoding( new org.hl7.fhir.r4.model.Coding() .setSystem(LIBRARY_TYPE_SYSTEM) - .setCode("sql-query"))); + .setCode(SQL_QUERY_TYPE_CODE))); library .addContent() .setContentType("application/sql") @@ -362,7 +417,7 @@ void acceptsSqlContentTypeWithDialect() { assertThat(result.getSql()).isEqualTo("SELECT 1"); } - /** Creates a minimal Library resource with the given SQL as Base64-encoded content. */ + /** Creates a minimal SQLQuery Library resource with the given SQL as Base64-encoded content. */ private Library createMinimalLibrary(final String sql) { final Library library = new Library(); library.setStatus(PublicationStatus.ACTIVE); @@ -378,6 +433,8 @@ private Library createMinimalLibrary(final String sql) { private static CodeableConcept sqlQueryTypeCoding() { return new CodeableConcept() .addCoding( - new org.hl7.fhir.r4.model.Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode("sql-query")); + new org.hl7.fhir.r4.model.Coding() + .setSystem(LIBRARY_TYPE_SYSTEM) + .setCode(SQL_QUERY_TYPE_CODE)); } } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java index 6f14c70622..ea33031001 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java @@ -138,7 +138,8 @@ void outputNameFollowsPrecedenceQueryNameThenLibraryNameThenGenerated() { final PreparedSqlQuery prepared = new PreparedSqlQuery( new SqlQueryRequest( - new ParsedSqlQuery("SELECT 1", List.of(), List.of()), + new ParsedSqlQuery( + "SELECT 1", List.of(), List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE), SqlQueryOutputFormat.NDJSON, true, null, @@ -174,7 +175,8 @@ private static SqlQueryExportRequest request( private static QueryInput queryInput(final String name) { final ParsedSqlQuery parsedQuery = - new ParsedSqlQuery("SELECT id FROM patients", List.of(), List.of()); + new ParsedSqlQuery( + "SELECT id FROM patients", List.of(), List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE); final SqlQueryRequest request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); return new QueryInput(name, null, new PreparedSqlQuery(request, Map.of())); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java index 385e4e6820..24ca1ae14f 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java @@ -232,9 +232,9 @@ private Map failingLibrary() { List.of( Map.of( "system", - SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM, + SqlLibraryParser.LIBRARY_TYPE_SYSTEM, "code", - SqlQueryLibraryParser.LIBRARY_TYPE_CODE)))); + SqlLibraryParser.SQL_QUERY_TYPE_CODE)))); final String sql = "SELECT nonexistent_column FROM patients"; library.put( "content", diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java index 6996486079..ade03ab139 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java @@ -322,9 +322,9 @@ private Map inlineSqlLibrary() { List.of( Map.of( "system", - SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM, + SqlLibraryParser.LIBRARY_TYPE_SYSTEM, "code", - SqlQueryLibraryParser.LIBRARY_TYPE_CODE)))); + SqlLibraryParser.SQL_QUERY_TYPE_CODE)))); final String sql = "SELECT id, family_name FROM patients ORDER BY id"; library.put( "content", diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java index 3cbfea5acd..4f04962708 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java @@ -70,7 +70,11 @@ void setUp() { deltaLake); final ParsedSqlQuery parsedQuery = - new ParsedSqlQuery("SELECT id FROM patients", java.util.List.of(), java.util.List.of()); + new ParsedSqlQuery( + "SELECT id FROM patients", + java.util.List.of(), + java.util.List.of(), + SqlLibraryParser.SQL_QUERY_TYPE_CODE); final SqlQueryRequest request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); when(pipeline.prepare(any(), any(), any(), any(), any(), any(), any())) diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java index 06dd6c1c61..e982903977 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java @@ -17,8 +17,8 @@ package au.csiro.pathling.operations.sqlquery; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_CODE; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.encoders.ViewDefinitionResource; @@ -182,7 +182,7 @@ static Library sqlLibrary( library.setStatus(PublicationStatus.ACTIVE); library.setType( new CodeableConcept() - .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(LIBRARY_TYPE_CODE))); + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); final Attachment content = new Attachment(); content.setContentType("application/sql"); content.setData(sql.getBytes(StandardCharsets.UTF_8)); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java index 320f95dcb2..de3ff94363 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java @@ -70,7 +70,8 @@ void setUp() { new ParsedSqlQuery( "SELECT id FROM patients", List.of(new ViewArtifactReference("patients", "ViewDefinition/patient-view")), - List.of()); + List.of(), + SqlLibraryParser.SQL_QUERY_TYPE_CODE); request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); final FhirView view = mock(FhirView.class); resolvedViews = Map.of("patients", view); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java index a318fb546c..0d6c6b7f3d 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java @@ -57,7 +57,7 @@ class SqlQueryRequestParserTest { @BeforeEach void setUp() { - parser = new SqlQueryRequestParser(new SqlQueryLibraryParser()); + parser = new SqlQueryRequestParser(new SqlLibraryParser()); } // --------------------------------------------------------------------------- diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java index d926e01b50..4ee2c9cc0a 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java @@ -17,8 +17,8 @@ package au.csiro.pathling.operations.sqlquery; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_CODE; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; import static org.assertj.core.api.Assertions.assertThat; import au.csiro.pathling.util.TestDataSetup; @@ -211,7 +211,7 @@ private Library sqlQueryLibrary( library.setStatus(PublicationStatus.ACTIVE); library.setType( new CodeableConcept() - .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(LIBRARY_TYPE_CODE))); + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); final Attachment content = new Attachment(); content.setContentType("application/sql"); content.setData(sql.getBytes(StandardCharsets.UTF_8)); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java index db61f3d0fb..4d5037bf5f 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunProviderIT.java @@ -17,8 +17,8 @@ package au.csiro.pathling.operations.sqlquery; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_CODE; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; import static org.assertj.core.api.Assertions.assertThat; import ca.uhn.fhir.context.FhirContext; @@ -456,7 +456,7 @@ private Library sqlQueryLibrary(@Nonnull final String sql) { library.setStatus(PublicationStatus.ACTIVE); library.setType( new CodeableConcept() - .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(LIBRARY_TYPE_CODE))); + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); final Attachment content = new Attachment(); content.setContentType("application/sql"); content.setData(sql.getBytes(StandardCharsets.UTF_8)); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunSecurityIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunSecurityIT.java index 4e6d738872..e2b54c0e40 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunSecurityIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunSecurityIT.java @@ -17,8 +17,8 @@ package au.csiro.pathling.operations.sqlquery; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_CODE; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; import static org.assertj.core.api.Assertions.assertThat; import ca.uhn.fhir.context.FhirContext; @@ -185,7 +185,7 @@ private Library sqlQueryLibrary(@Nonnull final String sql) { library.setStatus(PublicationStatus.ACTIVE); library.setType( new CodeableConcept() - .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(LIBRARY_TYPE_CODE))); + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); final Attachment content = new Attachment(); content.setContentType("application/sql"); content.setData(sql.getBytes(StandardCharsets.UTF_8)); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java index 9d1d60b7eb..c4a92dfa21 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java @@ -17,8 +17,8 @@ package au.csiro.pathling.operations.sqlquery; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_CODE; -import static au.csiro.pathling.operations.sqlquery.SqlQueryLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; import static org.assertj.core.api.Assertions.assertThat; import ca.uhn.fhir.context.FhirContext; @@ -232,7 +232,7 @@ private Library sqlQueryLibrary( library.setStatus(PublicationStatus.ACTIVE); library.setType( new CodeableConcept() - .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(LIBRARY_TYPE_CODE))); + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); final Attachment content = new Attachment(); content.setContentType("application/sql"); content.setData(sql.getBytes(StandardCharsets.UTF_8)); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java index c42e3985ba..bd5356effd 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java @@ -95,6 +95,49 @@ void resolveTempViewNameFallsBackToHashWhenSanitisedRequestIdIsEmpty() { assertThat(dashes).isNotEqualTo(slashes); } + @Test + void resolveTempViewNameDerivesFromCanonicalKeyNotLabel() { + // The temp view name is keyed by the resolved resource's canonical key, so a key carrying a + // slash and dash (ViewDefinition/patient-view) is sanitised into a legal Spark identifier. + final String name = + ViewRegistrationService.resolveTempViewName("req1", "ViewDefinition/patient-view"); + assertThat(name).startsWith("sqlquery_req1_").doesNotContain("/").doesNotContain("-"); + } + + @Test + void resolveTempViewNameGivesDistinctNamesToDistinctKeys() { + // Two nodes that happen to share a label but resolve to different resources are keyed by their + // distinct canonical keys, so their temp views never collide. + final String left = ViewRegistrationService.resolveTempViewName("req1", "ViewDefinition/a"); + final String right = ViewRegistrationService.resolveTempViewName("req1", "ViewDefinition/b"); + assertThat(left).isNotEqualTo(right); + } + + // --------------------------------------------------------------------------- + // SQLView materialisation. + // --------------------------------------------------------------------------- + + @Test + void buildSqlViewRewritesAgainstChildTempViewsBeforeRunning() { + // Materialise a child node under its canonical key, then build a SQLView whose SQL selects from + // a label that maps to that child. The SQL must be rewritten to the child's temp view name + // before running, so the SQLView observes the child's rows. + final Dataset childData = singleColumnDataset("value", List.of("x", "y")); + final String childKey = "ViewDefinition/child"; + final String childViewName = service.registerDataset(childKey, childData, "req1"); + try { + final ResolvedSqlView node = + new ResolvedSqlView("Library/parent", "SELECT value FROM t", Map.of("t", childKey)); + + final Dataset result = service.buildSqlView(node, Map.of(childKey, childViewName)); + + assertThat(result.collectAsList().stream().map(row -> row.getString(0)).toList()) + .containsExactlyInAnyOrder("x", "y"); + } finally { + service.dropViews(List.of(childViewName)); + } + } + // --------------------------------------------------------------------------- // SQL rewriting. // --------------------------------------------------------------------------- From acd52385b3ba0ffb65904c90953f5280722afa38 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 09:09:04 +1000 Subject: [PATCH 026/162] feat: Resolve and execute SQLView dependency graphs Introduce a recursive dependency resolver that turns a top-level query's relatedArtifact references into a topologically ordered graph of resolved ViewDefinition and SQLView nodes, disambiguating each reference (explicit type prefix, else ViewDefinition-first with a SQLView fallback) and deduplicating shared nodes. The executor materialises the graph bottom-up as request-scoped temp views, rewriting and validating each node's SQL before running the top-level query. A SQLQuery can now compose a stored SQLView. --- .../operations/sqlquery/PreparedSqlQuery.java | 13 +- .../sqlquery/SqlDependencyResolver.java | 293 ++++++++++++++++++ .../operations/sqlquery/SqlQueryExecutor.java | 120 +++++-- .../sqlquery/SqlQueryExportSupport.java | 30 +- .../operations/sqlquery/SqlQueryPipeline.java | 47 ++- .../sqlquery/ViewRegistrationService.java | 45 --- .../operations/sqlquery/ViewResolver.java | 161 ++++++---- .../sqlquery/RequestViewResolutionTest.java | 47 +-- .../sqlquery/SqlDependencyResolverTest.java | 246 +++++++++++++++ .../sqlquery/SqlQueryExportExecutorTest.java | 7 +- .../SqlQueryExportRequestParserTest.java | 4 +- .../sqlquery/SqlQueryPipelineTest.java | 48 +-- .../sqlquery/SqlViewRunProviderIT.java | 246 +++++++++++++++ .../sqlquery/SqlViewTestConfiguration.java | 174 +++++++++++ .../operations/sqlquery/ViewResolverTest.java | 97 +++--- 15 files changed, 1312 insertions(+), 266 deletions(-) create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java index a89abf1e49..61bbbfd434 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java @@ -17,14 +17,12 @@ package au.csiro.pathling.operations.sqlquery; -import au.csiro.pathling.views.FhirView; import jakarta.annotation.Nonnull; -import java.util.Map; import lombok.Value; /** - * A SQL query that has been parsed and had its ViewDefinition table sources resolved, ready for - * static validation and execution by {@link SqlQueryPipeline}. Produced by {@link + * A SQL query that has been parsed and had its dependency graph resolved, ready for static + * validation and execution by {@link SqlQueryPipeline}. Produced by {@link * SqlQueryPipeline#prepare}; shared by the synchronous {@code $sqlquery-run} and the asynchronous * {@code $sqlquery-export} operations. */ @@ -34,6 +32,9 @@ public class PreparedSqlQuery { /** The validated, normalised request: parsed query, output format, header flag, bindings. */ @Nonnull SqlQueryRequest request; - /** The resolved view table sources the SQL references, keyed by table label. */ - @Nonnull Map resolvedViews; + /** + * The resolved dependency graph the top-level SQL references: the transitive set of + * ViewDefinition and SQLView nodes, topologically ordered for materialisation. + */ + @Nonnull ResolvedDependencyGraph dependencyGraph; } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java new file mode 100644 index 0000000000..5ff9b85109 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java @@ -0,0 +1,293 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.security.PathlingAuthority; +import au.csiro.pathling.security.ResourceAccess.AccessType; +import au.csiro.pathling.security.SecurityAspect; +import au.csiro.pathling.views.FhirView; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import jakarta.annotation.Nonnull; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Reference; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Resolves the transitive dependency graph of a top-level query (a {@code SQLQuery} or a {@code + * SQLView}) into a {@link ResolvedDependencyGraph}, without touching Spark. Each {@code + * relatedArtifact} dependency is disambiguated, fetched, authorised, and parsed; {@code SQLView} + * dependencies are recursed into so the full graph of virtual tables is resolved. + * + *

    Reference disambiguation follows the SQL on FHIR resolution contract: + * + *

      + *
    • {@code ViewDefinition/[id]} resolves a {@code ViewDefinition} by logical id; + *
    • {@code Library/[id]} resolves a {@code SQLView} {@code Library} by logical id; + *
    • a bare canonical resolves a {@code ViewDefinition} first (by the canonical's final path + * segment as id), falling back to a {@code SQLView} {@code Library} by canonical url. + *
    + * + *

    The resolution memoises by canonical key, so a node referenced from more than one place (a + * diamond) is resolved once and shared. A reference encountered while it is already on the + * resolution stack is a cycle and is rejected, as is a graph that nests deeper than the configured + * {@code maxDependencyDepth}. All such failures are reported before any Spark execution. + * + * @author John Grimes + */ +@Component +public class SqlDependencyResolver { + + private static final String VIEW_DEFINITION_PREFIX = "ViewDefinition/"; + + private static final String LIBRARY_PREFIX = "Library/"; + + private static final String LIBRARY = "Library"; + + @Nonnull private final ViewResolver viewResolver; + + @Nonnull private final LibraryReferenceResolver libraryReferenceResolver; + + @Nonnull private final SqlLibraryParser libraryParser; + + @Nonnull private final ServerConfiguration serverConfiguration; + + /** + * Constructs a new SqlDependencyResolver. + * + * @param viewResolver resolves ViewDefinition leaves, preferring request-supplied views + * @param libraryReferenceResolver resolves a SQLView Library reference (relative or canonical) + * from storage + * @param libraryParser the shared parser for SQLView Libraries + * @param serverConfiguration the server configuration (auth toggle and the dependency depth cap) + */ + @Autowired + public SqlDependencyResolver( + @Nonnull final ViewResolver viewResolver, + @Nonnull final LibraryReferenceResolver libraryReferenceResolver, + @Nonnull final SqlLibraryParser libraryParser, + @Nonnull final ServerConfiguration serverConfiguration) { + this.viewResolver = viewResolver; + this.libraryReferenceResolver = libraryReferenceResolver; + this.libraryParser = libraryParser; + this.serverConfiguration = serverConfiguration; + } + + /** + * Resolves the dependency graph for a parsed top-level query. + * + * @param topLevel the parsed top-level query (SQLQuery or SQLView) + * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy, used + * for the top-level query's direct references; nested SQLView dependencies resolve from + * storage only + * @return the resolved dependency graph, topologically ordered + * @throws InvalidRequestException if a reference is unresolvable, a cycle or depth-limit breach + * is detected, or a dependency is a malformed or wrong-typed resource + */ + @Nonnull + public ResolvedDependencyGraph resolve( + @Nonnull final ParsedSqlQuery topLevel, @Nonnull final Map suppliedViews) { + final int maxDepth = serverConfiguration.getSqlQuery().getMaxDependencyDepth(); + final Map nodesByKey = new LinkedHashMap<>(); + final Set resolutionStack = new LinkedHashSet<>(); + final Map topLevelKeysByLabel = + resolveReferences( + topLevel.getViewReferences(), suppliedViews, 1, maxDepth, resolutionStack, nodesByKey); + return new ResolvedDependencyGraph( + new ArrayList<>(nodesByKey.values()), topLevelKeysByLabel, nodesByKey); + } + + /** + * Resolves a list of references in order, returning their labels mapped to the canonical keys of + * the nodes they resolve to. + */ + @Nonnull + private Map resolveReferences( + @Nonnull final java.util.List references, + @Nonnull final Map suppliedViews, + final int depth, + final int maxDepth, + @Nonnull final Set resolutionStack, + @Nonnull final Map nodesByKey) { + final Map keysByLabel = new LinkedHashMap<>(); + for (final ViewArtifactReference reference : references) { + keysByLabel.put( + reference.getLabel(), + resolveReference(reference, suppliedViews, depth, maxDepth, resolutionStack, nodesByKey)); + } + return keysByLabel; + } + + /** Resolves a single reference into the canonical key of its node, registering it if new. */ + @Nonnull + private String resolveReference( + @Nonnull final ViewArtifactReference reference, + @Nonnull final Map suppliedViews, + final int depth, + final int maxDepth, + @Nonnull final Set resolutionStack, + @Nonnull final Map nodesByKey) { + if (depth > maxDepth) { + throw new InvalidRequestException( + "Dependency graph nests deeper than the configured maximum of " + + maxDepth + + " (at label '" + + reference.getLabel() + + "', reference '" + + reference.getCanonicalUrl() + + "')"); + } + + final String referenceValue = reference.getCanonicalUrl(); + + if (referenceValue.startsWith(VIEW_DEFINITION_PREFIX)) { + // An explicit ViewDefinition reference is authoritative; no fallback. + return registerLeaf(viewResolver.resolveViewDefinition(reference, suppliedViews), nodesByKey); + } + if (referenceValue.startsWith(LIBRARY_PREFIX)) { + // An explicit Library reference is authoritative; resolve a SQLView, no fallback. + return resolveSqlView(reference, depth, maxDepth, resolutionStack, nodesByKey); + } + + // Bare canonical (or bare id): try a ViewDefinition first, then fall back to a SQLView. + final Optional viewDefinition = + viewResolver.tryResolveViewDefinition(reference, suppliedViews); + if (viewDefinition.isPresent()) { + return registerLeaf(viewDefinition.get(), nodesByKey); + } + return resolveSqlView(reference, depth, maxDepth, resolutionStack, nodesByKey); + } + + /** Registers a resolved ViewDefinition leaf (deduplicating diamonds) and returns its key. */ + @Nonnull + private String registerLeaf( + @Nonnull final ResolvedViewDefinition leaf, + @Nonnull final Map nodesByKey) { + nodesByKey.putIfAbsent(leaf.getCanonicalKey(), leaf); + return leaf.getCanonicalKey(); + } + + /** + * Resolves a reference to a {@code SQLView} {@code Library}, recursing into its own dependencies. + * Detects diamonds (already resolved), cycles (currently on the resolution stack), and rejects a + * {@code sql-query} Library referenced as a dependency. + */ + @Nonnull + private String resolveSqlView( + @Nonnull final ViewArtifactReference reference, + final int depth, + final int maxDepth, + @Nonnull final Set resolutionStack, + @Nonnull final Map nodesByKey) { + final Library library = readSqlViewLibrary(reference); + + // The Library was read from storage: enforce the metadata READ check. + if (serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority(PathlingAuthority.resourceAccess(AccessType.READ, LIBRARY)); + } + + final String canonicalKey = libraryCanonicalKey(library, reference); + + // A node already fully resolved is shared (diamond dedup). + if (nodesByKey.containsKey(canonicalKey)) { + return canonicalKey; + } + // A node still being resolved is a cycle. + if (resolutionStack.contains(canonicalKey)) { + throw new InvalidRequestException( + "Cyclic dependency detected: " + + String.join(" -> ", resolutionStack) + + " -> " + + canonicalKey); + } + + final ParsedSqlQuery parsed = libraryParser.parse(library); + if (!parsed.isView()) { + throw new InvalidRequestException( + "The dependency for label '" + + reference.getLabel() + + "' (reference '" + + reference.getCanonicalUrl() + + "') is a " + + parsed.getLibraryTypeCode() + + " Library, but only a SQLView may be referenced as a dependency"); + } + + resolutionStack.add(canonicalKey); + // Nested dependencies resolve from storage only; request-supplied views satisfy the top-level + // query's references, not the internals of a stored SQLView. + final Map childKeysByLabel = + resolveReferences( + parsed.getViewReferences(), Map.of(), depth + 1, maxDepth, resolutionStack, nodesByKey); + resolutionStack.remove(canonicalKey); + + final ResolvedSqlView node = + new ResolvedSqlView(canonicalKey, parsed.getSql(), childKeysByLabel); + nodesByKey.put(canonicalKey, node); + return canonicalKey; + } + + /** + * Reads the SQLView Library named by the reference from storage, wrapping any resolution failure + * in an actionable error that names the failing label and reference. + */ + @Nonnull + private Library readSqlViewLibrary(@Nonnull final ViewArtifactReference reference) { + final IBaseResource resource; + try { + resource = libraryReferenceResolver.resolve(new Reference(reference.getCanonicalUrl())); + } catch (final RuntimeException e) { + throw new InvalidRequestException( + "Failed to resolve the dependency for label '" + + reference.getLabel() + + "' with reference '" + + reference.getCanonicalUrl() + + "': " + + e.getMessage()); + } + if (resource instanceof final Library library) { + return library; + } + throw new InvalidRequestException( + "The dependency for label '" + + reference.getLabel() + + "' (reference '" + + reference.getCanonicalUrl() + + "') did not resolve to a Library"); + } + + /** + * Computes the canonical key for a resolved SQLView Library: its type and logical id, so two + * references to the same stored Library deduplicate. Falls back to the reference value when the + * resolved resource carries no logical id. + */ + @Nonnull + private String libraryCanonicalKey( + @Nonnull final Library library, @Nonnull final ViewArtifactReference reference) { + final String id = library.getIdElement() == null ? null : library.getIdElement().getIdPart(); + return LIBRARY_PREFIX + (id != null && !id.isBlank() ? id : reference.getCanonicalUrl()); + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java index 54e430efc8..fd307dc1b1 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java @@ -20,13 +20,12 @@ import au.csiro.pathling.config.ServerConfiguration; import au.csiro.pathling.config.SqlQueryConfiguration; import au.csiro.pathling.io.source.DataSource; -import au.csiro.pathling.views.FhirView; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.function.Consumer; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -35,9 +34,15 @@ import org.springframework.stereotype.Component; /** - * Executes the SQL of a {@link SqlQueryRequest} against the configured Spark session, owning the - * lifecycle of the request-scoped temporary views the query references. The only piece of the - * pipeline that touches Spark. + * Executes the SQL of a {@link SqlQueryRequest} against the configured Spark session, materialising + * the request's resolved dependency graph as request-scoped temporary views and owning their + * lifecycle. The only piece of the pipeline that touches Spark. + * + *

    Each node of the graph is materialised in topological order: a {@code ViewDefinition} leaf is + * executed as a view, and a {@code SQLView} node's SQL is rewritten against the temp views of its + * already-materialised children, validated, and run. The top-level SQL is then rewritten against + * its own direct dependencies' temp views and run. Every node's SQL is validated statically before + * execution and against its analysed plan during execution. */ @Slf4j @Component @@ -79,47 +84,66 @@ public SqlQueryExecutor( } /** - * Validates and executes the SQL, registering the resolved views under request-scoped temp view - * names for the duration of the call. The {@code consumer} is invoked with the result dataset - * before the temp views are dropped, so streaming and other terminal operations can complete - * before cleanup. + * Runs the static, read-only SQL validation for every node of the graph and the top-level query, + * each against its own declared labels, without touching Spark. Used both at export kick-off and + * before execution so malformed or disallowed SQL is caught early. + * + * @param request the parsed request + * @param graph the resolved dependency graph + */ + public void validateStatically( + @Nonnull final SqlQueryRequest request, @Nonnull final ResolvedDependencyGraph graph) { + for (final ResolvedDependency node : graph.getOrderedNodes()) { + if (node instanceof final ResolvedSqlView sqlView) { + sqlValidator.validate(sqlView.getSql(), sqlView.getChildKeysByLabel().keySet()); + } + } + sqlValidator.validate( + request.getParsedQuery().getSql(), graph.getTopLevelKeysByLabel().keySet()); + } + + /** + * Validates and executes the query, materialising the resolved dependency graph under + * request-scoped temp view names for the duration of the call. The {@code consumer} is invoked + * with the result dataset before the temp views are dropped, so streaming and other terminal + * operations can complete before cleanup. * * @param request the parsed and validated request - * @param resolvedViews the views referenced by the SQL, keyed by table label + * @param graph the resolved dependency graph the SQL references * @param dataSource the data source backing FhirView execution * @param requestId the HAPI per-request id used to namespace temp view names * @param consumer terminal consumer of the result dataset */ public void execute( @Nonnull final SqlQueryRequest request, - @Nonnull final Map resolvedViews, + @Nonnull final ResolvedDependencyGraph graph, @Nonnull final DataSource dataSource, @Nonnull final String requestId, @Nonnull final Consumer> consumer) { - final Set declaredLabels = - request.getParsedQuery().getViewReferences().stream() - .map(ViewArtifactReference::getLabel) - .collect(Collectors.toUnmodifiableSet()); - sqlValidator.validate(request.getParsedQuery().getSql(), declaredLabels); + validateStatically(request, graph); final String jobGroupId = "sqlquery-" + requestId; sparkSession .sparkContext() .setJobGroup(jobGroupId, "$sqlquery-run " + requestId, /* interruptOnCancel= */ true); - Map registeredViews = Map.of(); + final Map registeredByKey = new LinkedHashMap<>(); final SqlQueryWatchdog.Watch watch = watchdog.start(jobGroupId); try { - registeredViews = viewRegistrationService.registerViews(resolvedViews, dataSource, requestId); + for (final ResolvedDependency node : graph.getOrderedNodes()) { + materialiseNode(node, dataSource, requestId, registeredByKey); + } + final Map topLevelViews = + resolveLabelToViewName(graph.getTopLevelKeysByLabel(), registeredByKey); final String rewrittenSql = - viewRegistrationService.rewriteSql(request.getParsedQuery().getSql(), registeredViews); + viewRegistrationService.rewriteSql(request.getParsedQuery().getSql(), topLevelViews); Dataset result = runSql(rewrittenSql, request.getParameterBindings()); sqlValidator.validateAnalyzed( - result.queryExecution().analyzed(), Set.copyOf(registeredViews.values())); + result.queryExecution().analyzed(), Set.copyOf(topLevelViews.values())); result = result.limit(effectiveLimit(request.getLimit(), requestId)); @@ -135,8 +159,62 @@ public void execute( } finally { watch.complete(); sparkSession.sparkContext().clearJobGroup(); - viewRegistrationService.dropViews(registeredViews.values()); + viewRegistrationService.dropViews(registeredByKey.values()); + } + } + + /** + * Materialises a single graph node as a request-scoped temp view, recording its name by canonical + * key. A {@code SQLView} node's analysed plan is validated against its own children's temp views + * before registration, so it cannot reach an unauthorised data source. + */ + private void materialiseNode( + @Nonnull final ResolvedDependency node, + @Nonnull final DataSource dataSource, + @Nonnull final String requestId, + @Nonnull final Map registeredByKey) { + final Dataset dataset; + if (node instanceof final ResolvedViewDefinition viewDefinition) { + dataset = viewRegistrationService.buildViewDefinition(viewDefinition.getView(), dataSource); + } else if (node instanceof final ResolvedSqlView sqlView) { + dataset = viewRegistrationService.buildSqlView(sqlView, registeredByKey); + final Set childViewNames = + Set.copyOf( + resolveLabelToViewName(sqlView.getChildKeysByLabel(), registeredByKey).values()); + sqlValidator.validateAnalyzed(dataset.queryExecution().analyzed(), childViewNames); + } else { + throw new InvalidRequestException( + "Unsupported dependency node type: " + node.getClass().getSimpleName()); + } + final String tempViewName = + viewRegistrationService.registerDataset(node.getCanonicalKey(), dataset, requestId); + registeredByKey.put(node.getCanonicalKey(), tempViewName); + log.debug( + "Materialised temp view '{}' for dependency '{}'", tempViewName, node.getCanonicalKey()); + } + + /** + * Maps a node's local {@code label -> child canonical key} to {@code label -> temp view name}, + * resolving each child key against the views materialised so far. + */ + @Nonnull + private static Map resolveLabelToViewName( + @Nonnull final Map keysByLabel, + @Nonnull final Map registeredByKey) { + final Map labelToViewName = new LinkedHashMap<>(); + for (final Map.Entry entry : keysByLabel.entrySet()) { + final String viewName = registeredByKey.get(entry.getValue()); + if (viewName == null) { + throw new IllegalStateException( + "Dependency '" + + entry.getValue() + + "' for label '" + + entry.getKey() + + "' was not materialised before it was referenced"); + } + labelToViewName.put(entry.getKey(), viewName); } + return labelToViewName; } /** diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java index 3c8c7086b8..fb6accd423 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportSupport.java @@ -233,7 +233,8 @@ public String computeCacheKeyComponent(@Nonnull final SqlQueryExportRequest requ + ":" + q.preparedQuery().getRequest().getParsedQuery().getSql() + ":" - + gson.toJson(q.preparedQuery().getResolvedViews()) + + gson.toJson( + describeDependencyGraph(q.preparedQuery().getDependencyGraph())) + ":" + gson.toJson(q.preparedQuery().getRequest().getParameterBindings())) .collect(Collectors.joining(",")); @@ -256,6 +257,33 @@ public String computeCacheKeyComponent(@Nonnull final SqlQueryExportRequest requ return key.toString(); } + /** + * Renders a resolved dependency graph as a deterministic, serialisable description for the cache + * key, so two kick-offs whose composed queries differ deduplicate to distinct jobs. Captures the + * top-level label-to-node mapping and, for each node, its canonical key and (for a SQLView) its + * SQL and child label mapping. + */ + @Nonnull + private static List describeDependencyGraph( + @Nonnull final ResolvedDependencyGraph graph) { + final List parts = new java.util.ArrayList<>(); + parts.add("top=" + graph.getTopLevelKeysByLabel()); + for (final ResolvedDependency node : graph.getOrderedNodes()) { + if (node instanceof final ResolvedSqlView sqlView) { + parts.add( + "view:" + + sqlView.getCanonicalKey() + + ":" + + sqlView.getSql() + + ":" + + sqlView.getChildKeysByLabel()); + } else { + parts.add("vd:" + node.getCanonicalKey()); + } + } + return parts; + } + /** Collects patient ids from both the {@code patient} and {@code group} parameters. */ @Nonnull public Set collectPatientIds(@Nonnull final ServletRequestDetails requestDetails) { diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java index d994f5bbe2..097890d814 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipeline.java @@ -22,9 +22,7 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.Map; -import java.util.Set; import java.util.function.Consumer; -import java.util.stream.Collectors; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -52,32 +50,26 @@ public class SqlQueryPipeline { @Nonnull private final SqlQueryRequestParser requestParser; - @Nonnull private final ViewResolver viewResolver; + @Nonnull private final SqlDependencyResolver dependencyResolver; @Nonnull private final SqlQueryExecutor executor; - @Nonnull private final SqlValidator sqlValidator; - /** * Constructs a new SqlQueryPipeline. * - * @param requestParser parses a SQLQuery Library and binds runtime parameters - * @param viewResolver resolves the view references to parsed FhirViews, preferring - * request-supplied views over server storage + * @param requestParser parses a SQLQuery or SQLView Library and binds runtime parameters + * @param dependencyResolver resolves the transitive dependency graph, preferring request-supplied + * views over server storage * @param executor validates and runs the SQL against Spark - * @param sqlValidator the static SQL validator, used for kick-off-time validation that does not - * require executing the query */ @Autowired public SqlQueryPipeline( @Nonnull final SqlQueryRequestParser requestParser, - @Nonnull final ViewResolver viewResolver, - @Nonnull final SqlQueryExecutor executor, - @Nonnull final SqlValidator sqlValidator) { + @Nonnull final SqlDependencyResolver dependencyResolver, + @Nonnull final SqlQueryExecutor executor) { this.requestParser = requestParser; - this.viewResolver = viewResolver; + this.dependencyResolver = dependencyResolver; this.executor = executor; - this.sqlValidator = sqlValidator; } /** @@ -107,30 +99,27 @@ public PreparedSqlQuery prepare( @Nonnull final Map suppliedViews) { final SqlQueryRequest request = requestParser.parse(library, format, acceptHeader, includeHeader, limit, parameters); - final Map resolvedViews = - viewResolver.resolve(request.getParsedQuery().getViewReferences(), suppliedViews); - return new PreparedSqlQuery(request, resolvedViews); + final ResolvedDependencyGraph dependencyGraph = + dependencyResolver.resolve(request.getParsedQuery(), suppliedViews); + return new PreparedSqlQuery(request, dependencyGraph); } /** * Runs the static SQL validation that does not require executing the query, so that malformed or - * disallowed SQL is detected before any Spark work. Used by the asynchronous export to surface - * these failures synchronously at kick-off. + * disallowed SQL is detected before any Spark work. Validates the top-level SQL and every SQLView + * node's SQL against its own declared labels. Used by the asynchronous export to surface these + * failures synchronously at kick-off. * * @param prepared the prepared query */ public void validateStatically(@Nonnull final PreparedSqlQuery prepared) { - final Set declaredLabels = - prepared.getRequest().getParsedQuery().getViewReferences().stream() - .map(ViewArtifactReference::getLabel) - .collect(Collectors.toUnmodifiableSet()); - sqlValidator.validate(prepared.getRequest().getParsedQuery().getSql(), declaredLabels); + executor.validateStatically(prepared.getRequest(), prepared.getDependencyGraph()); } /** - * Executes the prepared query against Spark, registering the resolved views under request-scoped - * temp views for the duration of the call and invoking {@code consumer} with the result dataset - * before they are dropped. + * Executes the prepared query against Spark, materialising the resolved dependency graph under + * request-scoped temp views for the duration of the call and invoking {@code consumer} with the + * result dataset before they are dropped. * * @param prepared the prepared query * @param dataSource the data source backing FhirView execution (filtered for the export filters) @@ -143,6 +132,6 @@ public void execute( @Nonnull final String requestId, @Nonnull final Consumer> consumer) { executor.execute( - prepared.getRequest(), prepared.getResolvedViews(), dataSource, requestId, consumer); + prepared.getRequest(), prepared.getDependencyGraph(), dataSource, requestId, consumer); } } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java index d067b4dfa7..a38567b145 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java @@ -72,51 +72,6 @@ public ViewRegistrationService( this.queryConfiguration = serverConfiguration.getQuery(); } - /** - * Registers resolved ViewDefinitions as Spark temporary views, scoped by the request id so that - * concurrent executions cannot clobber one another in the session-global catalog. - * - * @param resolvedViews a map from label (table alias) to the parsed FhirView - * @param dataSource the data source to use for view execution - * @param requestId the per-request id used to namespace registered view names - * @return a map from original label to the actual temporary view name - */ - @Nonnull - public Map registerViews( - @Nonnull final Map resolvedViews, - @Nonnull final DataSource dataSource, - @Nonnull final String requestId) { - - final Map labelToViewName = new LinkedHashMap<>(); - - for (final Map.Entry entry : resolvedViews.entrySet()) { - final String label = entry.getKey(); - final FhirView view = entry.getValue(); - - final FhirViewExecutor executor = - new FhirViewExecutor(fhirContext, dataSource, queryConfiguration); - final Dataset result; - try { - result = executor.buildQuery(view); - } catch (final Exception e) { - // Drop any views that were already registered before failing. - dropViews(labelToViewName.values()); - throw new InvalidRequestException( - "Failed to execute ViewDefinition for label '" + label + "': " + e.getMessage()); - } - - final String tempViewName = registerDataset(label, result, requestId); - labelToViewName.put(label, tempViewName); - log.info( - "Registered temporary view '{}' for label '{}' (resource type '{}')", - tempViewName, - label, - view.getResource()); - } - - return labelToViewName; - } - /** * Registers an already-built dataset under a request-scoped temp view name derived from the given * identifier (a dependency's canonical key, or a bare label in the single-level tests) and diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java index b1355cb62d..6f8984d3cb 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java @@ -18,6 +18,7 @@ package au.csiro.pathling.operations.sqlquery; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.errors.ResourceNotFoundError; import au.csiro.pathling.read.ReadExecutor; import au.csiro.pathling.security.PathlingAuthority; import au.csiro.pathling.security.ResourceAccess.AccessType; @@ -29,17 +30,24 @@ import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import jakarta.annotation.Nonnull; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; +import java.util.Optional; import org.hl7.fhir.instance.model.api.IBaseResource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** - * Resolves the {@link ViewArtifactReference}s declared by a SQLQuery Library into parsed {@link - * FhirView}s, performing the per-resource-type authorisation check along the way. Has no Spark - * dependency; the FhirView is the parsed view configuration, not yet a {@code Dataset}. + * Resolves a single {@link ViewArtifactReference} that targets a {@code ViewDefinition} into a + * {@link ResolvedViewDefinition} leaf node, preferring a request-supplied view over server storage + * and performing the per-projected-resource-type authorisation check along the way. Has no Spark + * dependency; the {@link FhirView} is the parsed view configuration, not yet a {@code Dataset}. + * + *

    Used by {@link SqlDependencyResolver} for the {@code ViewDefinition} leaves of a dependency + * graph, both for explicit {@code ViewDefinition/[id]} references and as the first attempt when + * disambiguating a bare canonical reference (which falls back to a {@code SQLView} when no {@code + * ViewDefinition} matches). + * + * @author John Grimes */ // Bean name is set explicitly to avoid colliding with Spring MVC's DispatcherServlet, which // auto-discovers any bean named "viewResolver" and casts it to @@ -47,6 +55,8 @@ @Component("sqlQueryViewResolver") public class ViewResolver { + private static final String VIEW_DEFINITION = "ViewDefinition"; + @Nonnull private final ReadExecutor readExecutor; @Nonnull private final ServerConfiguration serverConfiguration; @@ -74,72 +84,88 @@ public ViewResolver( } /** - * Resolves each view reference to a parsed {@link FhirView}, reading every view from server - * storage. Equivalent to {@link #resolve(List, Map)} with no request-supplied views. + * Resolves a {@code ViewDefinition} reference into a {@link ResolvedViewDefinition}, throwing + * when no matching view is supplied or stored. Used for explicit {@code ViewDefinition/[id]} + * references, where the contract forbids falling back to another resource type. * - * @param references the references declared by the SQLQuery Library, keyed by table label - * @return a map from label to resolved FhirView - * @throws InvalidRequestException if a reference cannot be resolved or parsed + * @param reference the dependency reference to resolve + * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy + * @return the resolved leaf node + * @throws InvalidRequestException if the reference cannot be resolved or parsed */ @Nonnull - public Map resolve(@Nonnull final List references) { - return resolve(references, Map.of()); + public ResolvedViewDefinition resolveViewDefinition( + @Nonnull final ViewArtifactReference reference, + @Nonnull final Map suppliedViews) { + return tryResolveViewDefinition(reference, suppliedViews) + .orElseThrow( + () -> + new InvalidRequestException( + "Failed to resolve ViewDefinition for label '" + + reference.getLabel() + + "' with reference '" + + reference.getCanonicalUrl() + + "'")); } /** - * Resolves each view reference to a parsed {@link FhirView}, preserving label order. A - * request-supplied view is used in preference to server storage when it matches the reference (by - * the ViewDefinition id extracted from the reference); otherwise the view is read from server - * storage, exactly as {@code $sqlquery-run} does. Request-supplied views are assumed already - * parsed and, for stored references, authorisation-checked by the caller. + * Attempts to resolve a {@code ViewDefinition} reference into a {@link ResolvedViewDefinition}, + * returning empty when no request-supplied view satisfies it and no stored ViewDefinition with + * the reference's id exists. Used as the first step of disambiguating a bare canonical reference, + * which falls back to a {@code SQLView} when this returns empty. * - * @param references the references declared by the SQLQuery Library, keyed by table label - * @param suppliedViews request-supplied views keyed by the ViewDefinition id (or canonical url's - * final segment) they satisfy; may be empty - * @return a map from label to resolved FhirView - * @throws InvalidRequestException if a reference cannot be resolved or parsed + *

    A request-supplied view is preferred over storage and carries its own authorisation as part + * of the request payload. A stored view is subject to the per-projected-resource-type READ check + * (and, when authorisation is enabled, the {@code ViewDefinition} metadata READ check). + * + * @param reference the dependency reference to resolve + * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy + * @return the resolved leaf node, or empty if no ViewDefinition matches + * @throws InvalidRequestException if a stored ViewDefinition exists but cannot be parsed */ @Nonnull - public Map resolve( - @Nonnull final List references, + public Optional tryResolveViewDefinition( + @Nonnull final ViewArtifactReference reference, @Nonnull final Map suppliedViews) { - final Map resolved = new LinkedHashMap<>(); - - for (final ViewArtifactReference ref : references) { - final String viewDefinitionId = extractViewDefinitionId(ref.getCanonicalUrl()); - - // Prefer a request-supplied view that satisfies this reference, falling back to server - // storage when none is supplied. - final FhirView suppliedView = suppliedViews.get(viewDefinitionId); - if (suppliedView != null) { - resolved.put(ref.getLabel(), suppliedView); - continue; - } + final String viewDefinitionId = extractViewDefinitionId(reference.getCanonicalUrl()); + final String canonicalKey = VIEW_DEFINITION + "/" + viewDefinitionId; + + // Prefer a request-supplied view that satisfies this reference; it carries its own + // authorisation as the request payload and is not read from storage. + final FhirView suppliedView = suppliedViews.get(viewDefinitionId); + if (suppliedView != null) { + return Optional.of(new ResolvedViewDefinition(canonicalKey, suppliedView)); + } - final IBaseResource viewResource; - try { - viewResource = readExecutor.read("ViewDefinition", viewDefinitionId); - } catch (final Exception e) { - throw new InvalidRequestException( - "Failed to resolve ViewDefinition for label '" - + ref.getLabel() - + "' with reference '" - + ref.getCanonicalUrl() - + "': " - + e.getMessage()); - } + final Optional stored = readViewDefinition(viewDefinitionId); + if (stored.isEmpty()) { + return Optional.empty(); + } - final FhirView view = parseViewDefinition(viewResource); + // The ViewDefinition was read from storage: enforce the metadata READ check, then parse it and + // enforce the per-projected-resource READ check. + checkMetadataReadAuthority(VIEW_DEFINITION); + final FhirView view = parseViewDefinition(stored.get()); + checkProjectedResourceReadAuthority(view); + return Optional.of(new ResolvedViewDefinition(canonicalKey, view)); + } - if (serverConfiguration.getAuth().isEnabled()) { - SecurityAspect.checkHasAuthority( - PathlingAuthority.resourceAccess(AccessType.READ, view.getResource())); + /** + * Reads a stored ViewDefinition by its logical id, mapping a missing resource to an empty result + * so the caller can fall back to another resolution strategy. + */ + @Nonnull + private Optional readViewDefinition(@Nonnull final String id) { + try { + return Optional.of(readExecutor.read(VIEW_DEFINITION, id)); + } catch (final ResourceNotFoundError e) { + return Optional.empty(); + } catch (final IllegalArgumentException e) { + if (e.getMessage() != null && e.getMessage().contains("No data found for resource type")) { + return Optional.empty(); } - - resolved.put(ref.getLabel(), view); + throw e; } - - return resolved; } /** @@ -165,4 +191,27 @@ private FhirView parseViewDefinition(@Nonnull final IBaseResource viewResource) throw new InvalidRequestException("Invalid ViewDefinition: " + e.getMessage()); } } + + /** + * Enforces the metadata READ check for a resource resolved from storage, when authorisation is + * enabled. Reading a {@code ViewDefinition} out of the server requires READ authority on the + * {@code ViewDefinition} type itself, independent of the data the view projects. + */ + private void checkMetadataReadAuthority(@Nonnull final String resourceType) { + if (serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority( + PathlingAuthority.resourceAccess(AccessType.READ, resourceType)); + } + } + + /** + * Enforces the per-projected-resource-type READ check for a parsed view, when authorisation is + * enabled. + */ + private void checkProjectedResourceReadAuthority(@Nonnull final FhirView view) { + if (serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority( + PathlingAuthority.resourceAccess(AccessType.READ, view.getResource())); + } + } } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java index f99003b997..ca7fc466b6 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java @@ -18,6 +18,8 @@ package au.csiro.pathling.operations.sqlquery; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -32,7 +34,6 @@ import au.csiro.pathling.views.FhirView; import ca.uhn.fhir.context.FhirContext; import jakarta.annotation.Nonnull; -import java.util.List; import java.util.Map; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.StringType; @@ -69,16 +70,12 @@ void suppliedViewIsPreferredAndStorageIsNotConsulted() { .build(); final Map suppliedViews = Map.of("patient-bp", supplied); - final Map resolved = - resolver.resolve( - List.of(new ViewArtifactReference("patients", "ViewDefinition/patient-bp")), - suppliedViews); + final ResolvedViewDefinition resolved = + resolver.resolveViewDefinition( + new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), suppliedViews); - assertThat(resolved.get("patients")).isSameAs(supplied); - verify(readExecutor, never()) - .read( - org.mockito.ArgumentMatchers.eq("ViewDefinition"), - org.mockito.ArgumentMatchers.anyString()); + assertThat(resolved.getView()).isSameAs(supplied); + verify(readExecutor, never()).read(eq("ViewDefinition"), anyString()); } @Test @@ -86,36 +83,14 @@ void referenceWithNoSuppliedViewFallsBackToStorage() { when(readExecutor.read("ViewDefinition", "patient-bp")) .thenReturn(simpleViewDefinition("patient-bp", "Patient")); - final Map resolved = - resolver.resolve( - List.of(new ViewArtifactReference("patients", "ViewDefinition/patient-bp")), Map.of()); + final ResolvedViewDefinition resolved = + resolver.resolveViewDefinition( + new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), Map.of()); - assertThat(resolved).containsOnlyKeys("patients"); + assertThat(resolved.getView().getResource()).isEqualTo("Patient"); verify(readExecutor).read("ViewDefinition", "patient-bp"); } - @Test - void mixesSuppliedAndStorageResolvedViews() { - final FhirView supplied = - FhirView.ofResource("Patient") - .select(FhirView.columns(FhirView.column("id", "id"))) - .build(); - when(readExecutor.read("ViewDefinition", "obs-view")) - .thenReturn(simpleViewDefinition("obs-view", "Observation")); - - final Map resolved = - resolver.resolve( - List.of( - new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), - new ViewArtifactReference("obs", "ViewDefinition/obs-view")), - Map.of("patient-bp", supplied)); - - assertThat(resolved.get("patients")).isSameAs(supplied); - assertThat(resolved.get("obs").getResource()).isEqualTo("Observation"); - verify(readExecutor).read("ViewDefinition", "obs-view"); - verify(readExecutor, never()).read("ViewDefinition", "patient-bp"); - } - @Nonnull private static ViewDefinitionResource simpleViewDefinition( @Nonnull final String id, @Nonnull final String resourceType) { diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java new file mode 100644 index 0000000000..caf870ffbe --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java @@ -0,0 +1,246 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.config.AuthorizationConfiguration; +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.config.SqlQueryConfiguration; +import au.csiro.pathling.views.FhirView; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.hl7.fhir.r4.model.Library; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link SqlDependencyResolver} covering reference disambiguation, the resolved + * graph shape for a {@code SQLQuery -> SQLView -> ViewDefinition} chain, request-supplied view + * preference, and the structural rejections (cycles, depth, malformed and wrong-typed + * dependencies). + * + * @author John Grimes + */ +class SqlDependencyResolverTest { + + private ViewResolver viewResolver; + private LibraryReferenceResolver libraryReferenceResolver; + private ServerConfiguration serverConfiguration; + private SqlDependencyResolver resolver; + + @BeforeEach + void setUp() { + viewResolver = mock(ViewResolver.class); + libraryReferenceResolver = mock(LibraryReferenceResolver.class); + serverConfiguration = new ServerConfiguration(); + final AuthorizationConfiguration auth = new AuthorizationConfiguration(); + auth.setEnabled(false); + serverConfiguration.setAuth(auth); + serverConfiguration.setSqlQuery(new SqlQueryConfiguration()); + resolver = + new SqlDependencyResolver( + viewResolver, libraryReferenceResolver, new SqlLibraryParser(), serverConfiguration); + } + + // --------------------------------------------------------------------------- + // Disambiguation. + // --------------------------------------------------------------------------- + + @Test + void viewDefinitionPrefixResolvesAViewDefinition() { + stubViewDefinition("ViewDefinition/patient-view", "Patient"); + + final ResolvedDependencyGraph graph = + resolver.resolve(sqlQuery("SELECT * FROM p", "p", "ViewDefinition/patient-view"), Map.of()); + + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("p", "ViewDefinition/patient-view"); + assertThat(graph.getNodesByKey().get("ViewDefinition/patient-view")) + .isInstanceOf(ResolvedViewDefinition.class); + } + + @Test + void libraryPrefixResolvesASqlView() { + // Library/base is a SQLView over a ViewDefinition. + stubSqlView("base", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); + stubViewDefinition("ViewDefinition/patient-view", "Patient"); + + final ResolvedDependencyGraph graph = + resolver.resolve(sqlQuery("SELECT * FROM b", "b", "Library/base"), Map.of()); + + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("b", "Library/base"); + assertThat(graph.getNodesByKey().get("Library/base")).isInstanceOf(ResolvedSqlView.class); + assertThat(graph.getNodesByKey().get("ViewDefinition/patient-view")) + .isInstanceOf(ResolvedViewDefinition.class); + } + + @Test + void bareCanonicalResolvesViewDefinitionFirst() { + when(viewResolver.tryResolveViewDefinition( + argThat(ref -> "https://example.org/views/p".equals(ref.getCanonicalUrl())), any())) + .thenReturn( + Optional.of(new ResolvedViewDefinition("ViewDefinition/p", fhirView("Patient")))); + + final ResolvedDependencyGraph graph = + resolver.resolve(sqlQuery("SELECT * FROM p", "p", "https://example.org/views/p"), Map.of()); + + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("p", "ViewDefinition/p"); + assertThat(graph.getNodesByKey().get("ViewDefinition/p")) + .isInstanceOf(ResolvedViewDefinition.class); + } + + @Test + void bareCanonicalFallsBackToSqlViewWhenNoViewDefinition() { + final String canonical = "https://example.org/SQLView/active"; + when(viewResolver.tryResolveViewDefinition( + argThat(ref -> canonical.equals(ref.getCanonicalUrl())), any())) + .thenReturn(Optional.empty()); + final Library sqlView = SqlLibraryFixtures.sqlView("SELECT 1"); + sqlView.setId("active"); + sqlView.setUrl(canonical); + when(libraryReferenceResolver.resolve(argThat(ref -> canonical.equals(ref.getReference())))) + .thenReturn(sqlView); + + final ResolvedDependencyGraph graph = + resolver.resolve(sqlQuery("SELECT * FROM a", "a", canonical), Map.of()); + + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("a", "Library/active"); + assertThat(graph.getNodesByKey().get("Library/active")).isInstanceOf(ResolvedSqlView.class); + } + + @Test + void unresolvableReferenceErrorsNamingLabelAndResource() { + when(libraryReferenceResolver.resolve(any())) + .thenThrow(new ResourceNotFoundException("not found")); + + assertThatThrownBy( + () -> resolver.resolve(sqlQuery("SELECT 1", "x", "Library/missing"), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("x") + .hasMessageContaining("Library/missing"); + } + + @Test + void sqlQueryReferencedAsDependencyIsRejected() { + // A Library/q that is itself a sql-query (not a sql-view) cannot be a dependency. + final Library nested = SqlLibraryFixtures.sqlQuery("SELECT 1"); + nested.setId("q"); + when(libraryReferenceResolver.resolve(argThat(ref -> "Library/q".equals(ref.getReference())))) + .thenReturn(nested); + + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT 1", "q", "Library/q"), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("sql-query") + .hasMessageContaining("SQLView"); + } + + // --------------------------------------------------------------------------- + // Graph shape. + // --------------------------------------------------------------------------- + + @Test + void buildsTopologicallyOrderedTwoNodeGraph() { + stubSqlView("base", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); + stubViewDefinition("ViewDefinition/patient-view", "Patient"); + + final ResolvedDependencyGraph graph = + resolver.resolve(sqlQuery("SELECT * FROM b", "b", "Library/base"), Map.of()); + + // The ViewDefinition leaf must appear before the SQLView that depends on it. + assertThat(graph.getOrderedNodes()).hasSize(2); + assertThat(graph.getOrderedNodes().get(0).getCanonicalKey()) + .isEqualTo("ViewDefinition/patient-view"); + assertThat(graph.getOrderedNodes().get(1).getCanonicalKey()).isEqualTo("Library/base"); + + final ResolvedSqlView sqlView = (ResolvedSqlView) graph.getNodesByKey().get("Library/base"); + assertThat(sqlView.getChildKeysByLabel()).containsEntry("pv", "ViewDefinition/patient-view"); + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("b", "Library/base"); + } + + @Test + void prefersRequestSuppliedViewDefinitionOverStorage() { + final FhirView supplied = fhirView("Patient"); + when(viewResolver.resolveViewDefinition( + argThat(ref -> "ViewDefinition/patient-view".equals(ref.getCanonicalUrl())), + argThat(map -> map.containsKey("patient-view")))) + .thenReturn(new ResolvedViewDefinition("ViewDefinition/patient-view", supplied)); + + final ResolvedDependencyGraph graph = + resolver.resolve( + sqlQuery("SELECT * FROM p", "p", "ViewDefinition/patient-view"), + Map.of("patient-view", supplied)); + + final ResolvedViewDefinition node = + (ResolvedViewDefinition) graph.getNodesByKey().get("ViewDefinition/patient-view"); + assertThat(node.getView()).isSameAs(supplied); + } + + // --------------------------------------------------------------------------- + // Helpers. + // --------------------------------------------------------------------------- + + /** Builds a top-level SQLQuery ParsedSqlQuery with one dependency. */ + @Nonnull + private static ParsedSqlQuery sqlQuery( + @Nonnull final String sql, @Nonnull final String label, @Nonnull final String resource) { + return new ParsedSqlQuery( + sql, + List.of(new ViewArtifactReference(label, resource)), + List.of(), + SqlLibraryParser.SQL_QUERY_TYPE_CODE); + } + + /** Stubs the view resolver to resolve the given reference to a ViewDefinition over a resource. */ + private void stubViewDefinition( + @Nonnull final String reference, @Nonnull final String resourceType) { + final String key = + reference.startsWith("ViewDefinition/") ? reference : "ViewDefinition/" + reference; + when(viewResolver.resolveViewDefinition( + argThat(ref -> reference.equals(ref.getCanonicalUrl())), any())) + .thenReturn(new ResolvedViewDefinition(key, fhirView(resourceType))); + } + + /** Stubs the library resolver to return a stored SQLView with one dependency. */ + private void stubSqlView( + @Nonnull final String id, + @Nonnull final String sql, + @Nonnull final String depLabel, + @Nonnull final String depResource) { + final Library sqlView = SqlLibraryFixtures.sqlView(sql, depLabel, depResource); + sqlView.setId(id); + when(libraryReferenceResolver.resolve( + argThat(ref -> ("Library/" + id).equals(ref.getReference())))) + .thenReturn(sqlView); + } + + @Nonnull + private static FhirView fhirView(@Nonnull final String resourceType) { + return FhirView.ofResource(resourceType) + .select(FhirView.columns(FhirView.column("id", "id"))) + .build(); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java index ea33031001..f5b661a8ae 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportExecutorTest.java @@ -144,7 +144,7 @@ void outputNameFollowsPrecedenceQueryNameThenLibraryNameThenGenerated() { true, null, Map.of()), - Map.of()); + new ResolvedDependencyGraph(List.of(), Map.of(), Map.of())); // query.name wins. assertThat(new QueryInput("explicit", "lib", prepared).getEffectiveName(0)) @@ -179,6 +179,9 @@ private static QueryInput queryInput(final String name) { "SELECT id FROM patients", List.of(), List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE); final SqlQueryRequest request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); - return new QueryInput(name, null, new PreparedSqlQuery(request, Map.of())); + return new QueryInput( + name, + null, + new PreparedSqlQuery(request, new ResolvedDependencyGraph(List.of(), Map.of(), Map.of()))); } } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java index 4f04962708..34bfda7f11 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParserTest.java @@ -78,7 +78,9 @@ void setUp() { final SqlQueryRequest request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); when(pipeline.prepare(any(), any(), any(), any(), any(), any(), any())) - .thenReturn(new PreparedSqlQuery(request, Map.of())); + .thenReturn( + new PreparedSqlQuery( + request, new ResolvedDependencyGraph(java.util.List.of(), Map.of(), Map.of()))); when(libraryReferenceResolver.resolve(any())).thenReturn(new Library()); } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java index de3ff94363..90401804ed 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryPipelineTest.java @@ -29,7 +29,6 @@ import au.csiro.pathling.views.FhirView; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import org.apache.spark.sql.Dataset; @@ -39,31 +38,30 @@ import org.junit.jupiter.api.Test; /** - * Unit tests for {@link SqlQueryPipeline}, verifying that it orchestrates parsing, view resolution, - * static validation, and execution across its collaborators. The collaborators are mocked, so the - * test exercises the pipeline's wiring rather than the Spark-backed execution itself. + * Unit tests for {@link SqlQueryPipeline}, verifying that it orchestrates parsing, dependency + * resolution, static validation, and execution across its collaborators. The collaborators are + * mocked, so the test exercises the pipeline's wiring rather than the Spark-backed execution + * itself. * * @author John Grimes */ class SqlQueryPipelineTest { private SqlQueryRequestParser requestParser; - private ViewResolver viewResolver; + private SqlDependencyResolver dependencyResolver; private SqlQueryExecutor executor; - private SqlValidator sqlValidator; private SqlQueryPipeline pipeline; private Library library; private SqlQueryRequest request; - private Map resolvedViews; + private ResolvedDependencyGraph graph; @BeforeEach void setUp() { requestParser = mock(SqlQueryRequestParser.class); - viewResolver = mock(ViewResolver.class); + dependencyResolver = mock(SqlDependencyResolver.class); executor = mock(SqlQueryExecutor.class); - sqlValidator = mock(SqlValidator.class); - pipeline = new SqlQueryPipeline(requestParser, viewResolver, executor, sqlValidator); + pipeline = new SqlQueryPipeline(requestParser, dependencyResolver, executor); library = new Library(); final ParsedSqlQuery parsedQuery = @@ -73,40 +71,44 @@ void setUp() { List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE); request = new SqlQueryRequest(parsedQuery, SqlQueryOutputFormat.NDJSON, true, null, Map.of()); - final FhirView view = mock(FhirView.class); - resolvedViews = Map.of("patients", view); + final ResolvedViewDefinition leaf = + new ResolvedViewDefinition("ViewDefinition/patient-view", mock(FhirView.class)); + graph = + new ResolvedDependencyGraph( + List.of(leaf), + Map.of("patients", "ViewDefinition/patient-view"), + Map.of("ViewDefinition/patient-view", leaf)); } @Test - void prepareParsesAndResolvesViewsWithSuppliedViews() { + void prepareParsesAndResolvesDependencyGraphWithSuppliedViews() { final FhirView suppliedView = mock(FhirView.class); final Map supplied = Map.of("patient-view", suppliedView); when(requestParser.parse(eq(library), eq("ndjson"), any(), any(), any(), any())) .thenReturn(request); - when(viewResolver.resolve(request.getParsedQuery().getViewReferences(), supplied)) - .thenReturn(resolvedViews); + when(dependencyResolver.resolve(request.getParsedQuery(), supplied)).thenReturn(graph); final PreparedSqlQuery prepared = pipeline.prepare(library, "ndjson", null, null, null, null, supplied); assertThat(prepared.getRequest()).isSameAs(request); - assertThat(prepared.getResolvedViews()).isEqualTo(resolvedViews); + assertThat(prepared.getDependencyGraph()).isSameAs(graph); verify(requestParser).parse(eq(library), eq("ndjson"), any(), any(), any(), any()); - verify(viewResolver).resolve(request.getParsedQuery().getViewReferences(), supplied); + verify(dependencyResolver).resolve(request.getParsedQuery(), supplied); } @Test - void validateStaticallyValidatesSqlWithDeclaredLabels() { - final PreparedSqlQuery prepared = new PreparedSqlQuery(request, resolvedViews); + void validateStaticallyDelegatesToExecutor() { + final PreparedSqlQuery prepared = new PreparedSqlQuery(request, graph); pipeline.validateStatically(prepared); - verify(sqlValidator).validate("SELECT id FROM patients", Set.of("patients")); + verify(executor).validateStatically(request, graph); } @Test void executeDelegatesToExecutorAndPassesDatasetToConsumer() { - final PreparedSqlQuery prepared = new PreparedSqlQuery(request, resolvedViews); + final PreparedSqlQuery prepared = new PreparedSqlQuery(request, graph); final DataSource dataSource = mock(DataSource.class); @SuppressWarnings("unchecked") final Dataset dataset = mock(Dataset.class); @@ -119,12 +121,12 @@ void executeDelegatesToExecutorAndPassesDatasetToConsumer() { return null; }) .when(executor) - .execute(eq(request), eq(resolvedViews), eq(dataSource), eq("req-1"), any()); + .execute(eq(request), eq(graph), eq(dataSource), eq("req-1"), any()); final AtomicReference> received = new AtomicReference<>(); pipeline.execute(prepared, dataSource, "req-1", received::set); assertThat(received.get()).isSameAs(dataset); - verify(executor).execute(eq(request), eq(resolvedViews), eq(dataSource), eq("req-1"), any()); + verify(executor).execute(eq(request), eq(graph), eq(dataSource), eq("req-1"), any()); } } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java new file mode 100644 index 0000000000..3a3da21488 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java @@ -0,0 +1,246 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_QUERY_TYPE_CODE; +import static org.assertj.core.api.Assertions.assertThat; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.parser.IParser; +import com.google.gson.Gson; +import jakarta.annotation.Nonnull; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.Attachment; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.RelatedArtifact; +import org.hl7.fhir.r4.model.RelatedArtifact.RelatedArtifactType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.web.reactive.server.EntityExchangeResult; +import org.springframework.test.web.reactive.server.WebTestClient; + +/** + * End-to-end integration test for the {@code $sqlquery-run} operation against stored SQLView {@code + * Library} resources. Drives the full pipeline - dependency resolution, graph materialisation, and + * result streaming - for a SQLQuery that composes a SQLView (US1) and for nested and cyclic graphs + * (US2). + * + *

    Backed by {@link SqlViewTestConfiguration}, which substitutes an in-memory data source holding + * the ViewDefinitions, SQLViews, and FHIR data the queries resolve against. + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +@Import(SqlViewTestConfiguration.class) +class SqlViewRunProviderIT { + + private static final Gson GSON = new Gson(); + + @LocalServerPort int port; + + @Autowired WebTestClient webTestClient; + + @Autowired private FhirContext fhirContext; + + private IParser jsonParser; + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + } + + @BeforeEach + void setup() { + webTestClient = + webTestClient + .mutate() + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(100 * 1024 * 1024)) + // Nested SQLView graphs plan a deeper Spark query, so allow more than the 5s default. + .responseTimeout(java.time.Duration.ofSeconds(60)) + .build(); + jsonParser = fhirContext.newJsonParser(); + } + + @Test + void runsSqlQueryComposingAStoredSqlView() { + // A SQLQuery that selects from a SQLView (which itself selects from a ViewDefinition) returns + // the composed rows. + final Library library = + sqlQueryLibrary( + "SELECT id, family_name FROM ap ORDER BY id", + "ap", + "Library/" + SqlViewTestConfiguration.ACTIVE_PATIENTS_ID); + + final String body = postOk(parametersJson(library)); + + final String[] lines = body.trim().split("\n"); + assertThat(lines).hasSize(3); + assertThat(body) + .contains("\"id\":\"p1\"") + .contains("\"family_name\":\"Smith\"") + .contains("\"id\":\"p2\"") + .contains("\"family_name\":\"Johnson\"") + .contains("\"id\":\"p3\"") + .contains("\"family_name\":\"Williams\""); + } + + @Test + void runsSqlQueryComposingANestedSqlViewChain() { + // SQLQuery -> refined-patients (SQLView) -> active-patients (SQLView) -> patient-view (VD). + // refined-patients filters out Johnson, so two rows remain. + final Library library = + sqlQueryLibrary( + "SELECT id, family_name FROM rp ORDER BY id", + "rp", + "Library/" + SqlViewTestConfiguration.REFINED_PATIENTS_ID); + + final String body = postOk(parametersJson(library)); + + final String[] lines = body.trim().split("\n"); + assertThat(lines).hasSize(2); + assertThat(body).contains("Smith").contains("Williams").doesNotContain("Johnson"); + } + + @Test + void returns400WhenReferencedSqlViewDoesNotExist() { + final Library library = + sqlQueryLibrary("SELECT id FROM missing", "missing", "Library/does-not-exist"); + + // The error names the failing label and reference so the client can act on it. + final String body = postExpect4xx(parametersJson(library)); + assertThat(body).contains("missing").contains("does-not-exist"); + } + + @Test + void rejectsCyclicSqlViewGraphWith400() { + // cycle-a -> cycle-b -> cycle-a must be rejected before any SQL executes. + final Library library = + sqlQueryLibrary("SELECT * FROM a", "a", "Library/" + SqlViewTestConfiguration.CYCLE_A_ID); + + final String body = postExpect4xx(parametersJson(library)); + assertThat(body).containsIgnoringCase("cycl"); + } + + @Nonnull + private String postExpect4xx(@Nonnull final String body) { + final EntityExchangeResult result = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$sqlquery-run") + .header("Content-Type", "application/fhir+json") + .header("Accept", SqlQueryOutputFormat.NDJSON.getContentType()) + .bodyValue(body) + .exchange() + .expectStatus() + .is4xxClientError() + .expectBody() + .returnResult(); + final byte[] payload = result.getResponseBodyContent(); + return payload == null ? "" : new String(payload, StandardCharsets.UTF_8); + } + + @Nonnull + private String postOk(@Nonnull final String body) { + final EntityExchangeResult result = + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/$sqlquery-run") + .header("Content-Type", "application/fhir+json") + .header("Accept", SqlQueryOutputFormat.NDJSON.getContentType()) + .bodyValue(body) + .exchange() + .expectStatus() + .isOk() + .expectHeader() + .contentTypeCompatibleWith( + MediaType.parseMediaType(SqlQueryOutputFormat.NDJSON.getContentType())) + .expectBody() + .returnResult(); + return new String( + Objects.requireNonNull(result.getResponseBodyContent()), StandardCharsets.UTF_8); + } + + @Nonnull + private Library sqlQueryLibrary( + @Nonnull final String sql, @Nonnull final String label, @Nonnull final String resource) { + final Library library = new Library(); + library.setStatus(PublicationStatus.ACTIVE); + library.setType( + new CodeableConcept() + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); + final Attachment content = new Attachment(); + content.setContentType("application/sql"); + content.setData(sql.getBytes(StandardCharsets.UTF_8)); + library.addContent(content); + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel(label) + .setResource(resource)); + return library; + } + + @Nonnull + private String parametersJson(@Nonnull final Library library) { + final String libraryJson = jsonParser.encodeResourceToString(library); + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final List> parameterList = new ArrayList<>(); + + final Map queryResourceParam = new LinkedHashMap<>(); + queryResourceParam.put("name", "queryResource"); + queryResourceParam.put("resource", GSON.fromJson(libraryJson, Map.class)); + parameterList.add(queryResourceParam); + + final Map formatParam = new LinkedHashMap<>(); + formatParam.put("name", "_format"); + formatParam.put("valueString", SqlQueryOutputFormat.NDJSON.getCode()); + parameterList.add(formatParam); + + parameters.put("parameter", parameterList); + return GSON.toJson(parameters); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java new file mode 100644 index 0000000000..8e297ba47d --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java @@ -0,0 +1,174 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.LIBRARY_TYPE_SYSTEM; +import static au.csiro.pathling.operations.sqlquery.SqlLibraryParser.SQL_VIEW_TYPE_CODE; + +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; +import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; +import au.csiro.pathling.library.PathlingContext; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.util.CustomObjectDataSource; +import jakarta.annotation.Nonnull; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.spark.sql.SparkSession; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Attachment; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.RelatedArtifact; +import org.hl7.fhir.r4.model.RelatedArtifact.RelatedArtifactType; +import org.hl7.fhir.r4.model.StringType; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; + +/** + * Test configuration that overrides the production {@code deltaLake} data source with an in-memory + * one pre-loaded with FHIR resources, ViewDefinitions, and SQLView {@code Library} resources. Used + * by the SQLView end-to-end ITs to drive queries that resolve a SQLView dependency, a nested chain, + * a diamond, and a cycle, against real FHIR data. + * + *

    The stored graph is: + * + *

      + *
    • {@code ViewDefinition/patient-view} - Patient projection (id, family_name). + *
    • {@code Library/active-patients} - SQLView over {@code patient-view}. + *
    • {@code Library/refined-patients} - SQLView over {@code active-patients} (a nested chain). + *
    • {@code Library/cycle-a} / {@code Library/cycle-b} - a mutually-referencing cycle. + *
    + * + * @author John Grimes + */ +@TestConfiguration +public class SqlViewTestConfiguration { + + /** The id of the pre-loaded Patient ViewDefinition. */ + public static final String PATIENT_VIEW_ID = "patient-view"; + + /** The id of the SQLView over the Patient ViewDefinition. */ + public static final String ACTIVE_PATIENTS_ID = "active-patients"; + + /** The id of the SQLView over {@link #ACTIVE_PATIENTS_ID} (a nested chain). */ + public static final String REFINED_PATIENTS_ID = "refined-patients"; + + /** The id of one half of a mutually-referencing cycle. */ + public static final String CYCLE_A_ID = "cycle-a"; + + /** The id of the other half of a mutually-referencing cycle. */ + public static final String CYCLE_B_ID = "cycle-b"; + + @Primary + @Bean + @Nonnull + public QueryableDataSource deltaLake( + @Nonnull final SparkSession sparkSession, + @Nonnull final PathlingContext pathlingContext, + @Nonnull final FhirEncoders fhirEncoders) { + final List resources = new ArrayList<>(); + resources.add(patientView()); + resources.add( + sqlView( + ACTIVE_PATIENTS_ID, + "SELECT id, family_name FROM patient_view", + Map.of("patient_view", "ViewDefinition/" + PATIENT_VIEW_ID))); + resources.add( + sqlView( + REFINED_PATIENTS_ID, + "SELECT id, family_name FROM ap WHERE family_name <> 'Johnson'", + Map.of("ap", "Library/" + ACTIVE_PATIENTS_ID))); + resources.add(sqlView(CYCLE_A_ID, "SELECT * FROM b", Map.of("b", "Library/" + CYCLE_B_ID))); + resources.add(sqlView(CYCLE_B_ID, "SELECT * FROM a", Map.of("a", "Library/" + CYCLE_A_ID))); + resources.add(patient("p1", "Smith")); + resources.add(patient("p2", "Johnson")); + resources.add(patient("p3", "Williams")); + return new CustomObjectDataSource(sparkSession, pathlingContext, fhirEncoders, resources); + } + + @Nonnull + private static ViewDefinitionResource patientView() { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(PATIENT_VIEW_ID); + view.setName(new StringType("patient_view")); + view.setResource(new CodeType("Patient")); + view.setStatus(new CodeType("active")); + final SelectComponent select = new SelectComponent(); + select.getColumn().add(column("id", "id")); + select.getColumn().add(column("family_name", "name.first().family")); + view.getSelect().add(select); + return view; + } + + /** + * Builds a stored SQLView Library with the given id, SQL, and depends-on dependencies (label to + * resource reference, iteration order preserved). + */ + @Nonnull + private static Library sqlView( + @Nonnull final String id, + @Nonnull final String sql, + @Nonnull final Map dependenciesByLabel) { + final Library library = new Library(); + library.setId(id); + library.setUrl("https://pathling.csiro.au/test/Library/" + id); + library.setStatus(PublicationStatus.ACTIVE); + library.setType( + new CodeableConcept() + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_VIEW_TYPE_CODE))); + final Attachment content = new Attachment(); + content.setContentType("application/sql"); + content.setData(sql.getBytes(StandardCharsets.UTF_8)); + library.addContent(content); + new LinkedHashMap<>(dependenciesByLabel) + .forEach( + (label, resource) -> + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel(label) + .setResource(resource))); + return library; + } + + @Nonnull + private static ColumnComponent column(@Nonnull final String name, @Nonnull final String path) { + final ColumnComponent column = new ColumnComponent(); + column.setName(new StringType(name)); + column.setPath(new StringType(path)); + return column; + } + + @Nonnull + private static Patient patient(@Nonnull final String id, @Nonnull final String family) { + final Patient patient = new Patient(); + patient.setId(id); + patient.addName().setFamily(family); + return patient; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java index bde2660025..238671bb48 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java @@ -33,27 +33,26 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; -import java.util.List; import java.util.Map; +import java.util.Optional; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** - * Unit tests for {@link ViewResolver} covering id extraction, label-order preservation, and the - * read-error and parse-error wrapping paths. + * Unit tests for {@link ViewResolver} covering id extraction, the canonical key it produces, and + * the resolve-versus-try semantics for stored and missing ViewDefinitions. */ class ViewResolverTest { private ReadExecutor readExecutor; - private ServerConfiguration serverConfiguration; private ViewResolver resolver; @BeforeEach void setUp() { readExecutor = mock(ReadExecutor.class); - serverConfiguration = new ServerConfiguration(); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); final AuthorizationConfiguration auth = new AuthorizationConfiguration(); auth.setEnabled(false); serverConfiguration.setAuth(auth); @@ -61,69 +60,75 @@ void setUp() { } @Test - void resolvesEmptyReferenceListToEmptyMap() { - final Map resolved = resolver.resolve(List.of()); - assertThat(resolved).isEmpty(); - } - - @Test - void resolvesSingleReferenceByBareId() { + void resolvesReferenceByBareId() { when(readExecutor.read("ViewDefinition", "patient-view")) .thenReturn(simpleViewDefinition("patient-view", "Patient")); - final Map resolved = - resolver.resolve(List.of(new ViewArtifactReference("patients", "patient-view"))); + final ResolvedViewDefinition resolved = + resolver.resolveViewDefinition( + new ViewArtifactReference("patients", "patient-view"), Map.of()); - assertThat(resolved).containsOnlyKeys("patients"); - assertThat(resolved.get("patients").getResource()).isEqualTo("Patient"); + assertThat(resolved.getCanonicalKey()).isEqualTo("ViewDefinition/patient-view"); + assertThat(resolved.getView().getResource()).isEqualTo("Patient"); } @Test - void extractsIdFromCanonicalUrl() { + void extractsIdFromCanonicalUrlForTheKey() { when(readExecutor.read("ViewDefinition", "obs-view")) .thenReturn(simpleViewDefinition("obs-view", "Observation")); - final Map resolved = - resolver.resolve( - List.of( - new ViewArtifactReference("obs", "https://example.org/ViewDefinition/obs-view"))); + final ResolvedViewDefinition resolved = + resolver.resolveViewDefinition( + new ViewArtifactReference("obs", "https://example.org/ViewDefinition/obs-view"), + Map.of()); - assertThat(resolved).containsOnlyKeys("obs"); - assertThat(resolved.get("obs").getResource()).isEqualTo("Observation"); + assertThat(resolved.getCanonicalKey()).isEqualTo("ViewDefinition/obs-view"); + assertThat(resolved.getView().getResource()).isEqualTo("Observation"); } @Test - void preservesLabelOrderAcrossMultipleReferences() { - when(readExecutor.read("ViewDefinition", "a")).thenReturn(simpleViewDefinition("a", "Patient")); - when(readExecutor.read("ViewDefinition", "b")) - .thenReturn(simpleViewDefinition("b", "Observation")); - when(readExecutor.read("ViewDefinition", "c")) - .thenReturn(simpleViewDefinition("c", "Condition")); - - final Map resolved = - resolver.resolve( - List.of( - new ViewArtifactReference("first", "a"), - new ViewArtifactReference("second", "b"), - new ViewArtifactReference("third", "c"))); - - assertThat(resolved.keySet()).containsExactly("first", "second", "third"); - } - - @Test - void wrapsReadExecutorFailureWithLabelAndReference() { + void resolveThrowsWhenViewDefinitionNotFound() { when(readExecutor.read("ViewDefinition", "missing")) .thenThrow(new ResourceNotFoundError("not there")); - final List refs = - List.of(new ViewArtifactReference("patients", "missing")); - - assertThatThrownBy(() -> resolver.resolve(refs)) + assertThatThrownBy( + () -> + resolver.resolveViewDefinition( + new ViewArtifactReference("patients", "missing"), Map.of())) .isInstanceOf(InvalidRequestException.class) .hasMessageContaining("patients") .hasMessageContaining("missing"); } + @Test + void tryResolveReturnsEmptyWhenViewDefinitionNotFound() { + // The bare-canonical disambiguation relies on an empty result here to fall back to a SQLView. + when(readExecutor.read("ViewDefinition", "active-patients")) + .thenThrow(new ResourceNotFoundError("not there")); + + final Optional resolved = + resolver.tryResolveViewDefinition( + new ViewArtifactReference("ap", "active-patients"), Map.of()); + + assertThat(resolved).isEmpty(); + } + + @Test + void prefersSuppliedViewOverStorage() { + final FhirView supplied = + FhirView.ofResource("Patient") + .select(FhirView.columns(FhirView.column("id", "id"))) + .build(); + + final ResolvedViewDefinition resolved = + resolver.resolveViewDefinition( + new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), + Map.of("patient-bp", supplied)); + + assertThat(resolved.getView()).isSameAs(supplied); + assertThat(resolved.getCanonicalKey()).isEqualTo("ViewDefinition/patient-bp"); + } + @Nonnull private static ViewDefinitionResource simpleViewDefinition( @Nonnull final String id, @Nonnull final String resourceType) { From 822cf23c2fee4931ae1dad0612376391a26bd038 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 09:15:34 +1000 Subject: [PATCH 027/162] test: Cover nested SQLView graphs, diamonds, cycles, and depth Add resolver unit tests and end-to-end run coverage for the dependency-graph guarantees: a three-level nested chain resolves in topological order, a diamond shares its node once, the same label denotes different resources in different nodes without collision, and cyclic, self-referential, and over-deep graphs are rejected before any SQL executes. --- .../sqlquery/SqlDependencyResolverTest.java | 122 +++++++++++++++++- .../sqlquery/SqlViewRunProviderIT.java | 33 +++++ .../sqlquery/SqlViewTestConfiguration.java | 22 ++++ 3 files changed, 170 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java index caf870ffbe..f7ed22ad39 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java @@ -199,6 +199,108 @@ void prefersRequestSuppliedViewDefinitionOverStorage() { assertThat(node.getView()).isSameAs(supplied); } + // --------------------------------------------------------------------------- + // Nested graphs, diamonds, cycles, depth, and label scoping (US2). + // --------------------------------------------------------------------------- + + @Test + void resolvesAThreeLevelNestedChain() { + // SQLQuery -> v1 (SQLView) -> v2 (SQLView) -> ViewDefinition. + stubSqlView("v1", "SELECT * FROM x", "x", "Library/v2"); + stubSqlView("v2", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); + stubViewDefinition("ViewDefinition/patient-view", "Patient"); + + final ResolvedDependencyGraph graph = + resolver.resolve(sqlQuery("SELECT * FROM v", "v", "Library/v1"), Map.of()); + + assertThat(graph.getOrderedNodes()).hasSize(3); + // Dependencies precede dependents: VD, then v2, then v1. + assertThat(graph.getOrderedNodes().get(0).getCanonicalKey()) + .isEqualTo("ViewDefinition/patient-view"); + assertThat(graph.getOrderedNodes().get(1).getCanonicalKey()).isEqualTo("Library/v2"); + assertThat(graph.getOrderedNodes().get(2).getCanonicalKey()).isEqualTo("Library/v1"); + } + + @Test + void resolvesADiamondSharedNodeOnce() { + // SQLQuery references both left and right, each of which references the same shared SQLView. + stubSqlView("left", "SELECT * FROM s", "s", "Library/shared"); + stubSqlView("right", "SELECT * FROM s", "s", "Library/shared"); + stubSqlView("shared", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); + stubViewDefinition("ViewDefinition/patient-view", "Patient"); + + final ResolvedDependencyGraph graph = + resolver.resolve( + sqlQueryWithDeps( + "SELECT * FROM l JOIN r", Map.of("l", "Library/left", "r", "Library/right")), + Map.of()); + + // The shared node and the ViewDefinition each appear exactly once. + assertThat(graph.getNodesByKey()).containsKey("Library/shared"); + final long sharedCount = + graph.getOrderedNodes().stream() + .filter(node -> "Library/shared".equals(node.getCanonicalKey())) + .count(); + assertThat(sharedCount).isEqualTo(1); + assertThat(graph.getOrderedNodes()).hasSize(4); // shared, vd, left, right. + } + + @Test + void resolvesTheSameLabelInDifferentNodesWithoutCollision() { + // v1 and v2 both use label "t", but for different ViewDefinitions. + stubSqlView("v1", "SELECT * FROM t", "t", "ViewDefinition/a"); + stubSqlView("v2", "SELECT * FROM t", "t", "ViewDefinition/b"); + stubViewDefinition("ViewDefinition/a", "Patient"); + stubViewDefinition("ViewDefinition/b", "Observation"); + + final ResolvedDependencyGraph graph = + resolver.resolve( + sqlQueryWithDeps( + "SELECT * FROM one, two", Map.of("one", "Library/v1", "two", "Library/v2")), + Map.of()); + + final ResolvedSqlView v1 = (ResolvedSqlView) graph.getNodesByKey().get("Library/v1"); + final ResolvedSqlView v2 = (ResolvedSqlView) graph.getNodesByKey().get("Library/v2"); + assertThat(v1.getChildKeysByLabel()).containsEntry("t", "ViewDefinition/a"); + assertThat(v2.getChildKeysByLabel()).containsEntry("t", "ViewDefinition/b"); + } + + @Test + void rejectsACycleNamingTheChain() { + stubSqlView("a", "SELECT * FROM b", "b", "Library/b"); + stubSqlView("b", "SELECT * FROM a", "a", "Library/a"); + + assertThatThrownBy( + () -> resolver.resolve(sqlQuery("SELECT * FROM x", "x", "Library/a"), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContainingAll("Cyclic", "Library/a", "Library/b"); + } + + @Test + void rejectsASelfReference() { + stubSqlView("self", "SELECT * FROM s", "s", "Library/self"); + + assertThatThrownBy( + () -> resolver.resolve(sqlQuery("SELECT * FROM x", "x", "Library/self"), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("Cyclic"); + } + + @Test + void rejectsAGraphDeeperThanTheConfiguredLimit() { + serverConfiguration.getSqlQuery().setMaxDependencyDepth(2); + // top -> v1 (depth 1) -> v2 (depth 2) -> v3 (depth 3, exceeds the limit of 2). + stubSqlView("v1", "SELECT * FROM x", "x", "Library/v2"); + stubSqlView("v2", "SELECT * FROM y", "y", "Library/v3"); + stubSqlView("v3", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); + stubViewDefinition("ViewDefinition/patient-view", "Patient"); + + assertThatThrownBy( + () -> resolver.resolve(sqlQuery("SELECT * FROM v", "v", "Library/v1"), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContainingAll("deeper", "2"); + } + // --------------------------------------------------------------------------- // Helpers. // --------------------------------------------------------------------------- @@ -207,11 +309,17 @@ void prefersRequestSuppliedViewDefinitionOverStorage() { @Nonnull private static ParsedSqlQuery sqlQuery( @Nonnull final String sql, @Nonnull final String label, @Nonnull final String resource) { - return new ParsedSqlQuery( - sql, - List.of(new ViewArtifactReference(label, resource)), - List.of(), - SqlLibraryParser.SQL_QUERY_TYPE_CODE); + return sqlQueryWithDeps(sql, Map.of(label, resource)); + } + + /** Builds a top-level SQLQuery ParsedSqlQuery with several dependencies. */ + @Nonnull + private static ParsedSqlQuery sqlQueryWithDeps( + @Nonnull final String sql, @Nonnull final Map dependenciesByLabel) { + final List references = new java.util.ArrayList<>(); + dependenciesByLabel.forEach( + (label, resource) -> references.add(new ViewArtifactReference(label, resource))); + return new ParsedSqlQuery(sql, references, List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE); } /** Stubs the view resolver to resolve the given reference to a ViewDefinition over a resource. */ @@ -220,7 +328,7 @@ private void stubViewDefinition( final String key = reference.startsWith("ViewDefinition/") ? reference : "ViewDefinition/" + reference; when(viewResolver.resolveViewDefinition( - argThat(ref -> reference.equals(ref.getCanonicalUrl())), any())) + argThat(ref -> ref != null && reference.equals(ref.getCanonicalUrl())), any())) .thenReturn(new ResolvedViewDefinition(key, fhirView(resourceType))); } @@ -233,7 +341,7 @@ private void stubSqlView( final Library sqlView = SqlLibraryFixtures.sqlView(sql, depLabel, depResource); sqlView.setId(id); when(libraryReferenceResolver.resolve( - argThat(ref -> ("Library/" + id).equals(ref.getReference())))) + argThat(ref -> ref != null && ("Library/" + id).equals(ref.getReference())))) .thenReturn(sqlView); } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java index 3a3da21488..cd9d2dc5d1 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java @@ -144,6 +144,39 @@ void runsSqlQueryComposingANestedSqlViewChain() { assertThat(body).contains("Smith").contains("Williams").doesNotContain("Johnson"); } + @Test + void runsSqlQueryOverADiamondOfSqlViews() { + // left and right both depend on the shared SQLView; the join returns one row per patient, + // confirming both arms observe the same shared materialisation. + final Library library = new Library(); + library.setStatus(PublicationStatus.ACTIVE); + library.setType( + new CodeableConcept() + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode(SQL_QUERY_TYPE_CODE))); + final Attachment content = new Attachment(); + content.setContentType("application/sql"); + content.setData( + "SELECT l.id, l.family_name FROM l JOIN r ON l.id = r.id ORDER BY l.id" + .getBytes(StandardCharsets.UTF_8)); + library.addContent(content); + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel("l") + .setResource("Library/" + SqlViewTestConfiguration.LEFT_PATIENTS_ID)); + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel("r") + .setResource("Library/" + SqlViewTestConfiguration.RIGHT_PATIENTS_ID)); + + final String body = postOk(parametersJson(library)); + + final String[] lines = body.trim().split("\n"); + assertThat(lines).hasSize(3); + assertThat(body).contains("Smith").contains("Johnson").contains("Williams"); + } + @Test void returns400WhenReferencedSqlViewDoesNotExist() { final Library library = diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java index 8e297ba47d..3f98baa0b9 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java @@ -84,6 +84,15 @@ public class SqlViewTestConfiguration { /** The id of the other half of a mutually-referencing cycle. */ public static final String CYCLE_B_ID = "cycle-b"; + /** The id of the SQLView shared by both arms of a diamond. */ + public static final String SHARED_PATIENTS_ID = "shared-patients"; + + /** The id of the left arm of a diamond, over {@link #SHARED_PATIENTS_ID}. */ + public static final String LEFT_PATIENTS_ID = "left-patients"; + + /** The id of the right arm of a diamond, over {@link #SHARED_PATIENTS_ID}. */ + public static final String RIGHT_PATIENTS_ID = "right-patients"; + @Primary @Bean @Nonnull @@ -105,6 +114,19 @@ public QueryableDataSource deltaLake( Map.of("ap", "Library/" + ACTIVE_PATIENTS_ID))); resources.add(sqlView(CYCLE_A_ID, "SELECT * FROM b", Map.of("b", "Library/" + CYCLE_B_ID))); resources.add(sqlView(CYCLE_B_ID, "SELECT * FROM a", Map.of("a", "Library/" + CYCLE_A_ID))); + resources.add( + sqlView( + SHARED_PATIENTS_ID, + "SELECT id, family_name FROM patient_view", + Map.of("patient_view", "ViewDefinition/" + PATIENT_VIEW_ID))); + resources.add( + sqlView( + LEFT_PATIENTS_ID, + "SELECT id, family_name FROM sp", + Map.of("sp", "Library/" + SHARED_PATIENTS_ID))); + resources.add( + sqlView( + RIGHT_PATIENTS_ID, "SELECT id FROM sp", Map.of("sp", "Library/" + SHARED_PATIENTS_ID))); resources.add(patient("p1", "Smith")); resources.add(patient("p2", "Johnson")); resources.add(patient("p3", "Williams")); From ee3a0997cbe21eefb7331834c644cb8d6ef961b6 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 09:21:13 +1000 Subject: [PATCH 028/162] feat: Run and export a SQLView as a top-level resource A SQLView Library is now accepted as the top-level resource of $sqlquery-run and $sqlquery-export, at the system, type, and instance levels, executing as a parameter-less query. Supplying parameters with a parameter-less SQLView is rejected, as is a top-level Library whose type is neither sql-query nor sql-view. --- .../sqlquery/SqlQueryRequestParserTest.java | 44 +++++++++ .../sqlquery/SqlViewExportProviderIT.java | 91 +++++++++++++++++++ .../sqlquery/SqlViewRunProviderIT.java | 64 +++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewExportProviderIT.java diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java index 0d6c6b7f3d..6ff0c9659d 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRequestParserTest.java @@ -303,6 +303,50 @@ void fallsBackToNdjsonWhenAcceptHeaderDoesNotMatch() { assertThat(request.getOutputFormat()).isEqualTo(SqlQueryOutputFormat.NDJSON); } + // --------------------------------------------------------------------------- + // Top-level SQLView (US3). + // --------------------------------------------------------------------------- + + @Test + void acceptsTopLevelSqlViewLibrary() { + // A SQLView supplied as the top-level resource parses as a parameter-less query. + final Library library = SqlLibraryFixtures.sqlView("SELECT 1"); + + final SqlQueryRequest request = parser.parse(library, null, null, null, null, null); + + assertThat(request.getParsedQuery().isView()).isTrue(); + assertThat(request.getParameterBindings()).isEmpty(); + } + + @Test + void rejectsParametersSuppliedWithTopLevelSqlView() { + // A SQLView declares no parameter, so any supplied binding must be rejected. + final Library library = SqlLibraryFixtures.sqlView("SELECT 1"); + final Parameters params = new Parameters(); + params.addParameter().setName("min_age").setValue(new IntegerType(42)); + + assertThatThrownBy(() -> parser.parse(library, null, null, null, null, params)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("min_age"); + } + + @Test + void rejectsTopLevelLibraryWithUnknownType() { + // A Library whose type is neither sql-query nor sql-view is rejected. + final Library library = new Library(); + library.setStatus(PublicationStatus.ACTIVE); + library.setType( + new CodeableConcept() + .addCoding(new Coding().setSystem(LIBRARY_TYPE_SYSTEM).setCode("logic-library"))); + library + .addContent() + .setContentType("application/sql") + .setData("SELECT 1".getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> parser.parse(library, null, null, null, null, null)) + .isInstanceOf(InvalidRequestException.class); + } + /** Builds a minimal SQLQuery-typed Library carrying the given SQL. */ private static Library libraryWithSql(final String sql) { final Library library = new Library(); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewExportProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewExportProviderIT.java new file mode 100644 index 0000000000..e43315d3e0 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewExportProviderIT.java @@ -0,0 +1,91 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +/** + * End-to-end integration test for {@code $sqlquery-export} against stored SQLView {@code Library} + * resources, confirming that the asynchronous export resolves and materialises the same dependency + * graph as {@code $sqlquery-run} (US3 - run or export a SQLView directly). + * + *

    Backed by {@link SqlViewTestConfiguration}, which holds the ViewDefinitions, SQLViews, and + * FHIR data the queries resolve against. + * + * @author John Grimes + */ +@Slf4j +@Tag("IntegrationTest") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ResourceLock(value = "wiremock", mode = ResourceAccessMode.READ_WRITE) +@ActiveProfiles({"integration-test"}) +@Import(SqlViewTestConfiguration.class) +class SqlViewExportProviderIT extends AbstractSqlQueryExportIT { + + @DynamicPropertySource + static void configureProperties(final DynamicPropertyRegistry registry) { + final Path warehouseDir = + Path.of("src/test/resources/test-data/bulk/fhir/delta").toAbsolutePath(); + registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir); + } + + @Test + void instanceLevelExportOfStoredSqlViewProducesComposedRows() throws InterruptedException { + // Exporting a stored SQLView at the instance level resolves its ViewDefinition dependency and + // writes the composed rows, identically to running it. + final Map manifest = + exportToCompletion( + instanceLevelUri(SqlViewTestConfiguration.ACTIVE_PATIENTS_ID), emptyParameters()); + + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("completed"); + assertThat(paramsByName(manifest, "output")).hasSize(1); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content) + .contains("\"family_name\":\"Smith\"") + .contains("\"family_name\":\"Johnson\"") + .contains("\"family_name\":\"Williams\""); + } + + @Test + void typeLevelExportWithQueryReferenceToSqlViewProducesOutput() throws InterruptedException { + // A SQLView referenced as a top-level query at the type level exports identically. + final Map manifest = + exportToCompletion( + typeLevelUri(), storedQuery(SqlViewTestConfiguration.ACTIVE_PATIENTS_ID, null)); + + assertThat(findParamValue(manifest, "status", "valueCode")).isEqualTo("completed"); + assertThat(paramsByName(manifest, "output")).hasSize(1); + final String content = + download(partValue(paramsByName(manifest, "output").get(0), "location", "valueUri")); + assertThat(content).contains("\"family_name\":\"Smith\""); + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java index cd9d2dc5d1..dc7f11283e 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java @@ -177,6 +177,34 @@ void runsSqlQueryOverADiamondOfSqlViews() { assertThat(body).contains("Smith").contains("Johnson").contains("Williams"); } + @Test + void runsStoredSqlViewAtInstanceLevel() { + // A stored SQLView supplied as the top-level resource of the instance-level operation executes + // as a parameter-less query and returns its rows. + final String body = + getOk( + "/fhir/Library/" + + SqlViewTestConfiguration.ACTIVE_PATIENTS_ID + + "/$sqlquery-run?_format=ndjson"); + + final String[] lines = body.trim().split("\n"); + assertThat(lines).hasSize(3); + assertThat(body).contains("Smith").contains("Johnson").contains("Williams"); + } + + @Test + void runsSqlViewByQueryReferenceAtSystemLevel() { + // A SQLView supplied as a top-level queryReference at the system level executes and returns its + // rows. + final String body = + postOk( + queryReferenceParametersJson("Library/" + SqlViewTestConfiguration.ACTIVE_PATIENTS_ID)); + + final String[] lines = body.trim().split("\n"); + assertThat(lines).hasSize(3); + assertThat(body).contains("Smith").contains("Johnson").contains("Williams"); + } + @Test void returns400WhenReferencedSqlViewDoesNotExist() { final Library library = @@ -215,6 +243,42 @@ private String postExpect4xx(@Nonnull final String body) { return payload == null ? "" : new String(payload, StandardCharsets.UTF_8); } + @Nonnull + private String getOk(@Nonnull final String path) { + final EntityExchangeResult result = + webTestClient + .get() + .uri("http://localhost:" + port + path) + .header("Accept", SqlQueryOutputFormat.NDJSON.getContentType()) + .exchange() + .expectStatus() + .isOk() + .expectBody() + .returnResult(); + return new String( + Objects.requireNonNull(result.getResponseBodyContent()), StandardCharsets.UTF_8); + } + + @Nonnull + private String queryReferenceParametersJson(@Nonnull final String reference) { + final Map parameters = new LinkedHashMap<>(); + parameters.put("resourceType", "Parameters"); + final List> parameterList = new ArrayList<>(); + + final Map queryReferenceParam = new LinkedHashMap<>(); + queryReferenceParam.put("name", "queryReference"); + queryReferenceParam.put("valueReference", Map.of("reference", reference)); + parameterList.add(queryReferenceParam); + + final Map formatParam = new LinkedHashMap<>(); + formatParam.put("name", "_format"); + formatParam.put("valueString", SqlQueryOutputFormat.NDJSON.getCode()); + parameterList.add(formatParam); + + parameters.put("parameter", parameterList); + return GSON.toJson(parameters); + } + @Nonnull private String postOk(@Nonnull final String body) { final EntityExchangeResult result = From 80e8207a598907a17250ec740704fab194623187 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 09:33:01 +1000 Subject: [PATCH 029/162] feat: Require metadata READ to resolve stored views and queries With authorisation enabled, reading a ViewDefinition from storage now requires READ on ViewDefinition and reading a SQLView Library requires READ on Library, enforced at every storage-read seam - the standalone $view-run and $view-export operations, the $sqlquery dependency graph, and the top-level by-reference query - layered on top of the existing per-projected-resource checks. A resource supplied inline in the request body is exempt, as it is not read from storage. --- .../sqlquery/LibraryReferenceResolver.java | 19 +- .../sqlquery/SqlDependencyResolver.java | 17 +- .../operations/view/ViewExecutionHelper.java | 9 +- .../LibraryReferenceResolverTest.java | 17 +- .../operations/sqlquery/SqlQueryAuthTest.java | 205 ++++++++++++++++++ .../view/ViewExecutionHelperAuthTest.java | 141 ++++++++++++ 6 files changed, 393 insertions(+), 15 deletions(-) create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperAuthTest.java diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java index 335d0c1312..42f969af2c 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java @@ -17,10 +17,14 @@ package au.csiro.pathling.operations.sqlquery; +import au.csiro.pathling.config.ServerConfiguration; import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.errors.ResourceNotFoundError; import au.csiro.pathling.io.source.DataSource; import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.security.PathlingAuthority; +import au.csiro.pathling.security.ResourceAccess.AccessType; +import au.csiro.pathling.security.SecurityAspect; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import jakarta.annotation.Nonnull; @@ -60,25 +64,32 @@ public class LibraryReferenceResolver { @Nonnull private final FhirEncoders fhirEncoders; + @Nonnull private final ServerConfiguration serverConfiguration; + /** * Constructs a new LibraryReferenceResolver. * * @param readExecutor used for relative-reference reads * @param dataSource the data source used for canonical-reference search * @param fhirEncoders FHIR encoders used to decode the search result rows + * @param serverConfiguration the server configuration (used for the auth toggle) */ @Autowired public LibraryReferenceResolver( @Nonnull final ReadExecutor readExecutor, @Nonnull final DataSource dataSource, - @Nonnull final FhirEncoders fhirEncoders) { + @Nonnull final FhirEncoders fhirEncoders, + @Nonnull final ServerConfiguration serverConfiguration) { this.readExecutor = readExecutor; this.dataSource = dataSource; this.fhirEncoders = fhirEncoders; + this.serverConfiguration = serverConfiguration; } /** - * Resolves the reference to a stored Library resource. + * Resolves the reference to a stored Library resource. As the Library is read from server + * storage, the metadata READ check on {@code Library} is enforced when authorisation is enabled, + * regardless of the operation that triggered the read. * * @param reference the reference to resolve; must carry a non-blank {@code reference} value * @return the resolved Library resource @@ -93,6 +104,10 @@ public IBaseResource resolve(@Nonnull final Reference reference) { "queryReference must carry a non-blank Reference.reference value"); } + if (serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority(PathlingAuthority.resourceAccess(AccessType.READ, LIBRARY)); + } + if (isCanonical(ref)) { return resolveCanonical(ref); } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java index 5ff9b85109..9dec5782b4 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java @@ -18,9 +18,7 @@ package au.csiro.pathling.operations.sqlquery; import au.csiro.pathling.config.ServerConfiguration; -import au.csiro.pathling.security.PathlingAuthority; -import au.csiro.pathling.security.ResourceAccess.AccessType; -import au.csiro.pathling.security.SecurityAspect; +import au.csiro.pathling.errors.AccessDeniedError; import au.csiro.pathling.views.FhirView; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; @@ -65,8 +63,6 @@ public class SqlDependencyResolver { private static final String LIBRARY_PREFIX = "Library/"; - private static final String LIBRARY = "Library"; - @Nonnull private final ViewResolver viewResolver; @Nonnull private final LibraryReferenceResolver libraryReferenceResolver; @@ -202,13 +198,10 @@ private String resolveSqlView( final int maxDepth, @Nonnull final Set resolutionStack, @Nonnull final Map nodesByKey) { + // The Library is read from storage by LibraryReferenceResolver, which enforces the Library + // metadata READ check when authorisation is enabled. final Library library = readSqlViewLibrary(reference); - // The Library was read from storage: enforce the metadata READ check. - if (serverConfiguration.getAuth().isEnabled()) { - SecurityAspect.checkHasAuthority(PathlingAuthority.resourceAccess(AccessType.READ, LIBRARY)); - } - final String canonicalKey = libraryCanonicalKey(library, reference); // A node already fully resolved is shared (diamond dedup). @@ -259,6 +252,10 @@ private Library readSqlViewLibrary(@Nonnull final ViewArtifactReference referenc final IBaseResource resource; try { resource = libraryReferenceResolver.resolve(new Reference(reference.getCanonicalUrl())); + } catch (final AccessDeniedError e) { + // An authorisation denial must surface as a 403, not be reshaped into an unresolvable- + // reference 400. + throw e; } catch (final RuntimeException e) { throw new InvalidRequestException( "Failed to resolve the dependency for label '" diff --git a/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java b/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java index 1c798858c8..fac47030e0 100644 --- a/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java +++ b/server/src/main/java/au/csiro/pathling/operations/view/ViewExecutionHelper.java @@ -185,7 +185,10 @@ public IBaseResource resolveViewInput( } /** - * Reads a stored ViewDefinition by its logical id, mapping a missing resource to a 404. + * Reads a stored ViewDefinition by its logical id, mapping a missing resource to a 404. As the + * ViewDefinition is read from server storage, the metadata READ check on {@code ViewDefinition} + * is enforced when authorisation is enabled, layered on top of the per-projected-resource check + * applied later when the view is executed. * * @param id the logical id of the stored ViewDefinition * @return the stored ViewDefinition resource @@ -193,6 +196,10 @@ public IBaseResource resolveViewInput( */ @Nonnull public IBaseResource readStoredViewDefinition(@Nonnull final String id) { + if (serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority( + PathlingAuthority.resourceAccess(AccessType.READ, "ViewDefinition")); + } try { return readExecutor.read("ViewDefinition", id); } catch (final ResourceNotFoundError e) { diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java index 426cbabfda..2ab8298896 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java @@ -68,7 +68,7 @@ void setUp() { readExecutor = mock(ReadExecutor.class); resolver = new LibraryReferenceResolver( - readExecutor, mock(DataSource.class), mock(FhirEncoders.class)); + readExecutor, mock(DataSource.class), mock(FhirEncoders.class), authDisabledConfig()); } @Test @@ -140,7 +140,9 @@ class CanonicalReferences { @BeforeEach void setUp() { dataSource = mock(DataSource.class); - resolver = new LibraryReferenceResolver(mock(ReadExecutor.class), dataSource, fhirEncoders); + resolver = + new LibraryReferenceResolver( + mock(ReadExecutor.class), dataSource, fhirEncoders, authDisabledConfig()); } @Test @@ -239,6 +241,17 @@ private Dataset libraryDataset(final Library... libraries) { // Helpers shared across nested classes. // --------------------------------------------------------------------------- + /** Builds a server configuration with authorisation disabled, so no metadata READ is enforced. */ + private static au.csiro.pathling.config.ServerConfiguration authDisabledConfig() { + final au.csiro.pathling.config.ServerConfiguration config = + new au.csiro.pathling.config.ServerConfiguration(); + final au.csiro.pathling.config.AuthorizationConfiguration auth = + new au.csiro.pathling.config.AuthorizationConfiguration(); + auth.setEnabled(false); + config.setAuth(auth); + return config; + } + private static Library newLibrary( final String id, final String url, final String version, final PublicationStatus status) { final Library library = new Library(); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java new file mode 100644 index 0000000000..bd15a8b82c --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java @@ -0,0 +1,205 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.config.AuthorizationConfiguration; +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.config.SqlQueryConfiguration; +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; +import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; +import au.csiro.pathling.errors.AccessDeniedError; +import au.csiro.pathling.io.source.DataSource; +import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.views.FhirView; +import ca.uhn.fhir.context.FhirContext; +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.Map; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.security.core.authority.AuthorityUtils; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; + +/** + * Security tests for the {@code $sqlquery-*} resolution path, wiring the real {@link ViewResolver}, + * {@link LibraryReferenceResolver}, and {@link SqlDependencyResolver} with authorisation enabled. + * Verifies the metadata-resource authorisation matrix: a stored ViewDefinition dependency requires + * {@code ViewDefinition} READ, a stored SQLView dependency requires {@code Library} READ, the + * per-projected-resource READ still applies at each leaf, and a request-supplied (inline) view + * requires no metadata READ. + * + * @author John Grimes + */ +@Tag("UnitTest") +class SqlQueryAuthTest { + + private ReadExecutor readExecutor; + private LibraryReferenceResolver libraryReferenceResolver; + private SqlDependencyResolver resolver; + + @BeforeEach + void setUp() { + readExecutor = mock(ReadExecutor.class); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + final AuthorizationConfiguration auth = new AuthorizationConfiguration(); + auth.setEnabled(true); + serverConfiguration.setAuth(auth); + serverConfiguration.setSqlQuery(new SqlQueryConfiguration()); + + final ViewResolver viewResolver = + new ViewResolver(readExecutor, serverConfiguration, FhirContext.forR4Cached()); + libraryReferenceResolver = + new LibraryReferenceResolver( + readExecutor, mock(DataSource.class), mock(FhirEncoders.class), serverConfiguration); + resolver = + new SqlDependencyResolver( + viewResolver, libraryReferenceResolver, new SqlLibraryParser(), serverConfiguration); + } + + @AfterEach + void tearDown() { + SecurityContextHolder.clearContext(); + } + + @Test + void storedViewDefinitionDependencyRequiresViewDefinitionRead() { + when(readExecutor.read("ViewDefinition", "pv")) + .thenReturn(simpleViewDefinition("pv", "Patient")); + + // Projected-resource READ alone is not enough; the ViewDefinition metadata READ is required. + setSecurityContext("pathling:read:Patient"); + assertThatThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of())) + .isInstanceOf(AccessDeniedError.class) + .hasMessageContaining("ViewDefinition"); + + setSecurityContext("pathling:read:ViewDefinition", "pathling:read:Patient"); + assertThatNoException() + .isThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of())); + } + + @Test + void projectedResourceReadStillRequiredAtTheLeaf() { + when(readExecutor.read("ViewDefinition", "pv")) + .thenReturn(simpleViewDefinition("pv", "Patient")); + + // ViewDefinition READ without the projected Patient READ is still denied. + setSecurityContext("pathling:read:ViewDefinition"); + assertThatThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of())) + .isInstanceOf(AccessDeniedError.class) + .hasMessageContaining("Patient"); + } + + @Test + void storedSqlViewDependencyRequiresLibraryRead() { + final Library base = SqlLibraryFixtures.sqlView("SELECT * FROM pv", "pv", "ViewDefinition/pv"); + base.setId("base"); + when(readExecutor.read("Library", "base")).thenReturn(base); + when(readExecutor.read("ViewDefinition", "pv")) + .thenReturn(simpleViewDefinition("pv", "Patient")); + + // Holding the transitive ViewDefinition and projected reads, but not Library READ, is denied. + setSecurityContext("pathling:read:ViewDefinition", "pathling:read:Patient"); + assertThatThrownBy(() -> resolver.resolve(sqlQuery("Library/base"), Map.of())) + .isInstanceOf(AccessDeniedError.class) + .hasMessageContaining("Library"); + + setSecurityContext( + "pathling:read:Library", "pathling:read:ViewDefinition", "pathling:read:Patient"); + assertThatNoException().isThrownBy(() -> resolver.resolve(sqlQuery("Library/base"), Map.of())); + } + + @Test + void inlineSuppliedViewRequiresNoMetadataRead() { + // A request-supplied (inline) view is not read from storage, so resolving it needs no metadata + // READ - even with no authorities granted. + setSecurityContext("pathling:sqlquery-run"); + final FhirView supplied = + FhirView.ofResource("Patient") + .select(FhirView.columns(FhirView.column("id", "id"))) + .build(); + + assertThatNoException() + .isThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of("pv", supplied))); + } + + @Test + void topLevelQueryReferenceRequiresLibraryRead() { + final Library base = SqlLibraryFixtures.sqlView("SELECT 1"); + base.setId("base"); + when(readExecutor.read("Library", "base")).thenReturn(base); + + // The top-level by-reference resolution goes through LibraryReferenceResolver, which enforces + // the Library metadata READ. + setSecurityContext("pathling:sqlquery-run"); + assertThatThrownBy(() -> libraryReferenceResolver.resolve(new Reference("Library/base"))) + .isInstanceOf(AccessDeniedError.class) + .hasMessageContaining("Library"); + + setSecurityContext("pathling:read:Library"); + assertThat(libraryReferenceResolver.resolve(new Reference("Library/base"))).isNotNull(); + } + + @Nonnull + private static ParsedSqlQuery sqlQuery(@Nonnull final String resource) { + return new ParsedSqlQuery( + "SELECT * FROM t", + List.of(new ViewArtifactReference("t", resource)), + List.of(), + SqlLibraryParser.SQL_QUERY_TYPE_CODE); + } + + private void setSecurityContext(final String... authorities) { + final Jwt jwt = Jwt.withTokenValue("mock").header("alg", "none").claim("sub", "user").build(); + final JwtAuthenticationToken auth = + new JwtAuthenticationToken(jwt, AuthorityUtils.createAuthorityList(authorities)); + SecurityContextHolder.getContext().setAuthentication(auth); + } + + @Nonnull + private static ViewDefinitionResource simpleViewDefinition( + @Nonnull final String id, @Nonnull final String resourceType) { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(id); + view.setName(new StringType(id + "_view")); + view.setResource(new CodeType(resourceType)); + view.setStatus(new CodeType("active")); + final SelectComponent select = new SelectComponent(); + final ColumnComponent column = new ColumnComponent(); + column.setName(new StringType("id")); + column.setPath(new StringType("id")); + select.getColumn().add(column); + view.getSelect().add(select); + return view; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperAuthTest.java b/server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperAuthTest.java new file mode 100644 index 0000000000..7730eae8a4 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/view/ViewExecutionHelperAuthTest.java @@ -0,0 +1,141 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.view; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import au.csiro.pathling.config.AuthorizationConfiguration; +import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; +import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; +import au.csiro.pathling.errors.AccessDeniedError; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.compartment.GroupMemberService; +import au.csiro.pathling.operations.compartment.PatientCompartmentService; +import au.csiro.pathling.read.ReadExecutor; +import ca.uhn.fhir.context.FhirContext; +import jakarta.annotation.Nonnull; +import org.apache.spark.sql.SparkSession; +import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.security.core.authority.AuthorityUtils; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; + +/** + * Security tests for {@link ViewExecutionHelper}, verifying that resolving a stored {@code + * ViewDefinition} from a {@code viewReference} requires the {@code ViewDefinition} metadata READ + * authority, while an inline {@code viewResource} requires no such check. + * + * @author John Grimes + */ +@Tag("UnitTest") +class ViewExecutionHelperAuthTest { + + private ReadExecutor readExecutor; + private AuthorizationConfiguration authConfig; + private ViewExecutionHelper helper; + + @BeforeEach + void setUp() { + readExecutor = mock(ReadExecutor.class); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + authConfig = new AuthorizationConfiguration(); + authConfig.setEnabled(true); + serverConfiguration.setAuth(authConfig); + helper = + new ViewExecutionHelper( + mock(SparkSession.class), + mock(QueryableDataSource.class), + FhirContext.forR4Cached(), + mock(FhirEncoders.class), + mock(PatientCompartmentService.class), + mock(GroupMemberService.class), + serverConfiguration, + readExecutor); + } + + @AfterEach + void tearDown() { + SecurityContextHolder.clearContext(); + } + + @Test + void viewReferenceRequiresViewDefinitionReadAuthority() { + setSecurityContext("pathling:read:Patient"); + when(readExecutor.read("ViewDefinition", "stored")) + .thenReturn(simpleViewDefinition("stored", "Patient")); + + assertThatThrownBy(() -> helper.resolveViewInput(null, new Reference("ViewDefinition/stored"))) + .isInstanceOf(AccessDeniedError.class) + .hasMessageContaining("ViewDefinition"); + } + + @Test + void viewReferenceSucceedsWithViewDefinitionReadAuthority() { + setSecurityContext("pathling:read:ViewDefinition"); + when(readExecutor.read("ViewDefinition", "stored")) + .thenReturn(simpleViewDefinition("stored", "Patient")); + + assertThat(helper.resolveViewInput(null, new Reference("ViewDefinition/stored"))).isNotNull(); + } + + @Test + void inlineViewResourceRequiresNoViewDefinitionRead() { + // An inline viewResource is not read from storage, so the metadata READ check does not apply. + setSecurityContext("pathling:search"); + final ViewDefinitionResource inline = simpleViewDefinition("inline", "Patient"); + + assertThat(helper.resolveViewInput(inline, null)).isSameAs(inline); + } + + private void setSecurityContext(final String... authorities) { + final Jwt jwt = Jwt.withTokenValue("mock").header("alg", "none").claim("sub", "user").build(); + final JwtAuthenticationToken auth = + new JwtAuthenticationToken(jwt, AuthorityUtils.createAuthorityList(authorities)); + SecurityContextHolder.getContext().setAuthentication(auth); + } + + @Nonnull + private static ViewDefinitionResource simpleViewDefinition( + @Nonnull final String id, @Nonnull final String resourceType) { + final ViewDefinitionResource view = new ViewDefinitionResource(); + view.setId(id); + view.setName(new StringType(id + "_view")); + view.setResource(new CodeType(resourceType)); + view.setStatus(new CodeType("active")); + final SelectComponent select = new SelectComponent(); + final ColumnComponent column = new ColumnComponent(); + column.setName(new StringType("id")); + column.setPath(new StringType("id")); + select.getColumn().add(column); + view.getSelect().add(select); + return view; + } +} From d58f75f9955f1a97533f60958d53af7cac8568a4 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 09:38:16 +1000 Subject: [PATCH 030/162] docs: Document SQLView composition, depth limit, and authorisation Describe SQLView dependencies and top-level SQLViews on the run and export operations, the reference-resolution and disambiguation rules, the cycle/depth rejections, and the metadata-resource READ requirements. Document the maxDependencyDepth configuration option and the ViewDefinition/Library READ authorities, with the inline-versus-stored distinction. Mark authorship on the new and significantly changed files. --- .../sqlquery/LibraryReferenceResolver.java | 2 + .../operations/sqlquery/ParsedSqlQuery.java | 2 + .../operations/sqlquery/PreparedSqlQuery.java | 2 + .../operations/sqlquery/SqlQueryExecutor.java | 2 + .../sqlquery/ViewRegistrationService.java | 8 ++- site/docs/server/authorization.md | 16 +++++ site/docs/server/configuration.md | 6 ++ site/docs/server/operations/sql-export.md | 9 +++ site/docs/server/operations/sql-run.md | 58 +++++++++++++++++-- 9 files changed, 98 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java index 42f969af2c..176a81cede 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java @@ -52,6 +52,8 @@ *

  • Canonical references such as {@code https://example.org/Library/foo} or {@code * https://example.org/Library/foo|1.2}, matched against {@code Library.url}. * + * + * @author John Grimes */ @Component public class LibraryReferenceResolver { diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java index a60e0f8dc0..6f8acdb99b 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ParsedSqlQuery.java @@ -24,6 +24,8 @@ /** * Represents a parsed SQL on FHIR Library resource ({@code SQLQuery} or {@code SQLView}) containing * the SQL text, dependency references, declared parameters, and the resolved library type code. + * + * @author John Grimes */ @Value public class ParsedSqlQuery { diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java index 61bbbfd434..12b2963613 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/PreparedSqlQuery.java @@ -25,6 +25,8 @@ * validation and execution by {@link SqlQueryPipeline}. Produced by {@link * SqlQueryPipeline#prepare}; shared by the synchronous {@code $sqlquery-run} and the asynchronous * {@code $sqlquery-export} operations. + * + * @author John Grimes */ @Value public class PreparedSqlQuery { diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java index fd307dc1b1..49e3a44acc 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExecutor.java @@ -43,6 +43,8 @@ * already-materialised children, validated, and run. The top-level SQL is then rewritten against * its own direct dependencies' temp views and run. Every node's SQL is validated statically before * execution and against its analysed plan during execution. + * + * @author John Grimes */ @Slf4j @Component diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java index a38567b145..ffdd761b2c 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationService.java @@ -38,8 +38,12 @@ /** * Manages the lifecycle of Spark temporary views for SQL query execution. Each execution scopes its - * views to the HAPI per-request id so that concurrent {@code $sqlquery-run} requests using the same - * label cannot clobber one another in Spark's session-global temporary view catalog. + * views to the HAPI per-request id and keys them by the resolved dependency's canonical identity, + * so concurrent {@code $sqlquery-run} requests cannot clobber one another in Spark's session-global + * temporary view catalog, a shared node materialises once, and the same table label used in + * different nodes cannot collide. + * + * @author John Grimes */ @Slf4j @Component diff --git a/site/docs/server/authorization.md b/site/docs/server/authorization.md index a237c5dae0..85be6f9001 100644 --- a/site/docs/server/authorization.md +++ b/site/docs/server/authorization.md @@ -90,6 +90,22 @@ the resource type specified in the ViewDefinition's `resource` element. For example, a ViewDefinition targeting `Patient` resources requires `pathling:read:Patient` authority in addition to the operation authority. +### Reading view definitions and queries from storage + +Resolving a metadata resource from server storage requires `read` authority on +that resource type, in addition to the per-projected-resource checks above: + +- Resolving a `ViewDefinition` from storage requires `pathling:read:ViewDefinition`. + This applies to a `view-run` or `view-export` `viewReference`, and to a + `ViewDefinition` referenced as a dependency of a SQL query. +- Resolving a `SQLView` (a `Library`) from storage requires `pathling:read:Library`. + This applies to a `$sqlquery-run` or `$sqlquery-export` `queryReference` or + instance-level Library, and to a `SQLView` referenced as a dependency. + +A resource supplied inline in the request body (an inline `viewResource` or +`queryResource`) is not read from storage and is therefore not subject to these +metadata read checks, though the per-projected-resource read checks still apply. + ## SMART configuration When authorisation is enabled, Pathling exposes a diff --git a/site/docs/server/configuration.md b/site/docs/server/configuration.md index 393ef97110..76322d3594 100644 --- a/site/docs/server/configuration.md +++ b/site/docs/server/configuration.md @@ -183,6 +183,12 @@ limits are always applied; they cannot be disabled per request. 4xx response; a timeout that fires mid-stream aborts the connection and is recorded as a server warning. Long-running queries should use the asynchronous path. +- `pathling.sqlQuery.maxDependencyDepth` - (default: `10`) The maximum nesting + depth of the SQLView dependency graph resolved for a single query. The + top-level query's direct dependencies sit at depth one; each further level of + nested SQLView dependency increments the depth. A graph nested deeper is + rejected with a `400` before any Spark work, guarding against accidental + fan-out and runaway resolution. ### Encoding diff --git a/site/docs/server/operations/sql-export.md b/site/docs/server/operations/sql-export.md index a5a84a836f..8a91294f78 100644 --- a/site/docs/server/operations/sql-export.md +++ b/site/docs/server/operations/sql-export.md @@ -70,6 +70,15 @@ or `view.viewResource`; supplying both, or neither, returns `400 Bad Request`. A supplied ViewDefinition that is well-formed but semantically invalid returns `422 Unprocessable Entity`. +A `relatedArtifact` dependency may reference a SQLView as well as a +ViewDefinition, and a SQLView may itself depend on further ViewDefinitions and +SQLViews. A SQLView may also be the top-level resource of a `query` (or the +bound Library at the instance level), running as a parameter-less query. The +dependency-graph resolution, reference disambiguation, cycle and depth limits, +and metadata-resource authorisation are identical to the synchronous operation; +see [Composing SQLViews](./sql-run.md#composing-sqlviews). These structural +rejections are returned synchronously at kick-off. + The `source` parameter (an external data source) is not supported by this server; supplying it returns `400 Bad Request`. All of these rejections are returned synchronously at kick-off. diff --git a/site/docs/server/operations/sql-run.md b/site/docs/server/operations/sql-run.md index 0711ce334b..98aac8a0f0 100644 --- a/site/docs/server/operations/sql-run.md +++ b/site/docs/server/operations/sql-run.md @@ -66,15 +66,19 @@ The `$sqlquery-run` operation expects a Library that conforms to the SQLQuery profile. The relevant elements are: - `type` - must include the coding `sql-query` from - `https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes`. + `https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes`. A + [SQLView](https://build.fhir.org/ig/FHIR/sql-on-fhir-v2/StructureDefinition-SQLView.html) + (`sql-view`) is also accepted as a top-level resource and runs as a + parameter-less query. - `content` - exactly one entry with `contentType` of `application/sql` and the SQL text Base64-encoded in `data`. -- `relatedArtifact` - one entry per ViewDefinition the query references. The - `label` becomes the table name available to the SQL, and `resource` points to - the ViewDefinition (relative literal or canonical reference). +- `relatedArtifact` - one entry per dependency the query references. The `label` + becomes the table name available to the SQL, and `resource` points to a + ViewDefinition or a SQLView (relative literal or canonical reference). See + [Composing SQLViews](#composing-sqlviews). - `parameter` - optional declarations of named runtime parameters. Each entry with `use` of `in` must have a `name` and `type`, and the type must be a - primitive FHIR type. + primitive FHIR type. A SQLView declares no parameters. Example Library: @@ -273,6 +277,47 @@ time via the `parameters` input: Each binding's name must match a declaration on the Library, and its `value[x]` must match the declared FHIR type. +## Composing SQLViews + +A `relatedArtifact` dependency may reference a +[SQLView](https://build.fhir.org/ig/FHIR/sql-on-fhir-v2/StructureDefinition-SQLView.html) +as well as a ViewDefinition. A SQLView is a reusable, named SQL query +(a `Library` with `type` of `sql-view` and no parameters) that other queries +build on as a virtual table. A SQLView may itself depend on ViewDefinitions and +other SQLViews, forming a directed acyclic graph of virtual tables that the +server resolves and executes. Each node's result is available to its referrer +under the `label` of the `relatedArtifact` that points at it; labels are scoped +to the node that declares them, so the same label may name different sources in +different nodes. + +A SQLView may also be supplied directly as the top-level `queryResource`, +`queryReference`, or instance-level Library; it then executes as a +parameter-less query and returns its rows. + +### Reference resolution + +Each `relatedArtifact.resource` is resolved as follows: + +- `ViewDefinition/[id]` resolves a ViewDefinition by logical id. +- `Library/[id]` resolves a SQLView Library by logical id. +- A bare canonical (`[url]` or `[url]|[version]`) resolves a ViewDefinition + first, by the canonical's final path segment as id; if none exists, it falls + back to a SQLView Library matched by canonical `url`. + +An explicit relative type prefix is authoritative - the server does not fall +back to the other type when a prefixed reference fails to resolve. A reference +that resolves to neither, a `Library` that is a `sql-query` rather than a +`sql-view`, a cycle (for example `A -> B -> A`), and a graph nested deeper than +`pathling.sqlQuery.maxDependencyDepth` are each rejected with a `400` before any +SQL executes, with a message identifying the cause. + +When authorisation is enabled, resolving a ViewDefinition from storage requires +READ on `ViewDefinition`, and resolving a SQLView from storage requires READ on +`Library`, in addition to the READ check on each projected resource type. A +resource supplied inline in the request body is not read from storage and is not +subject to the metadata check. See the +[authorisation documentation](../authorization.md). + ## Resource limits Two server-configured limits are always applied to a `$sqlquery-run` @@ -283,6 +328,9 @@ invocation, regardless of any caller-supplied parameters: - `pathling.sqlQuery.timeoutSeconds` (default `60`) - the maximum wall-clock time, in seconds, that a query may run before its Spark job group is cancelled. +- `pathling.sqlQuery.maxDependencyDepth` (default `10`) - the maximum nesting + depth of the SQLView dependency graph. A graph nested deeper is rejected with + a `400` before any Spark work. Long-running queries should use the asynchronous bulk submit path rather than the synchronous `$sqlquery-run` endpoint. See the From ce0a55da69825263f481c00d6bace46c8095a121 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 10:03:49 +1000 Subject: [PATCH 031/162] refactor: Prefer imports over inline fully-qualified names Import java.util.List and java.util.ArrayList in the dependency resolver and its test rather than referencing them inline, for a cleaner reading. --- .../pathling/operations/sqlquery/SqlDependencyResolver.java | 3 ++- .../operations/sqlquery/SqlDependencyResolverTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java index 9dec5782b4..78b0e365bf 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -122,7 +123,7 @@ public ResolvedDependencyGraph resolve( */ @Nonnull private Map resolveReferences( - @Nonnull final java.util.List references, + @Nonnull final List references, @Nonnull final Map suppliedViews, final int depth, final int maxDepth, diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java index f7ed22ad39..4328067c7f 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java @@ -31,6 +31,7 @@ import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import jakarta.annotation.Nonnull; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -316,7 +317,7 @@ private static ParsedSqlQuery sqlQuery( @Nonnull private static ParsedSqlQuery sqlQueryWithDeps( @Nonnull final String sql, @Nonnull final Map dependenciesByLabel) { - final List references = new java.util.ArrayList<>(); + final List references = new ArrayList<>(); dependenciesByLabel.forEach( (label, resource) -> references.add(new ViewArtifactReference(label, resource))); return new ParsedSqlQuery(sql, references, List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE); From 743a034cca631770dcd71434c30cc01564c715be Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:10:55 +1000 Subject: [PATCH 032/162] test: Add SQLView Library fixtures for the SQL query UI Add a stored SQLView Library bundle fixture (and empty variant) mirroring the existing SQLQuery fixtures, in preparation for surfacing SQLViews in the SQL query form. --- ui/e2e/fixtures/fhirData.ts | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/ui/e2e/fixtures/fhirData.ts b/ui/e2e/fixtures/fhirData.ts index 938e977dfd..834c37b2ae 100644 --- a/ui/e2e/fixtures/fhirData.ts +++ b/ui/e2e/fixtures/fhirData.ts @@ -415,6 +415,72 @@ export const mockEmptySqlQueryLibraryBundle: Bundle = { entry: [], }; +// ============================================================================ +// SQLView (sql-view) Library mocks +// ============================================================================ + +/** + * Mock SQLView Library for use in `$sqlquery-run` / `$sqlquery-export` tests. + * + * Carries Base64-encoded SQL ("SELECT 2"), references a single ViewDefinition + * as a dependency and declares no parameters (SQLViews are parameter-less). + */ +export const mockSqlViewLibrary1 = { + resourceType: "Library", + id: "view-active-patients", + status: "active", + title: "Active patients", + type: { + coding: [ + { + system: "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes", + code: "sql-view", + }, + ], + }, + content: [ + { + contentType: "application/sql", + data: "U0VMRUNUIDI=", + }, + ], + relatedArtifact: [ + { + type: "depends-on", + label: "patients", + resource: "ViewDefinition/patient-demographics", + }, + ], +}; + +/** + * Mock Bundle containing SQLView Library search results. + */ +export const mockSqlViewLibraryBundle: Bundle = { + resourceType: "Bundle", + type: "searchset", + total: 1, + entry: [ + { + resource: mockSqlViewLibrary1 as Bundle["entry"] extends (infer T)[] + ? T extends { resource?: infer R } + ? R + : never + : never, + }, + ], +}; + +/** + * Mock empty SQLView Library Bundle for testing the empty-views state. + */ +export const mockEmptySqlViewLibraryBundle: Bundle = { + resourceType: "Bundle", + type: "searchset", + total: 0, + entry: [], +}; + /** * Mock CSV body for `$sqlquery-run` results. */ From 48e23c229288b72d94c2099d7bd5d99d53974693 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:14:52 +1000 Subject: [PATCH 033/162] feat: List stored SQLViews through a shared Library list core Generalise the SQLQuery Library list into listStoredLibraries(typeCode), add the sql-view token filter, and expose a useSqlViews hook so the SQL query form can fetch SQLViews alongside SQLQueries. SQLQuery listing keeps its existing query key and behaviour by delegating to the shared core. --- ui/src/api/__tests__/sqlQuery.test.ts | 84 +++++++++ ui/src/api/index.ts | 3 + ui/src/api/sqlQuery.ts | 67 ++++++- ui/src/hooks/__tests__/useSqlViews.test.tsx | 197 ++++++++++++++++++++ ui/src/hooks/index.ts | 2 + ui/src/hooks/useSqlQueryLibraries.ts | 5 +- ui/src/hooks/useSqlViews.ts | 72 +++++++ 7 files changed, 420 insertions(+), 10 deletions(-) create mode 100644 ui/src/hooks/__tests__/useSqlViews.test.tsx create mode 100644 ui/src/hooks/useSqlViews.ts diff --git a/ui/src/api/__tests__/sqlQuery.test.ts b/ui/src/api/__tests__/sqlQuery.test.ts index 3cd2d097fd..2e0ab34ab9 100644 --- a/ui/src/api/__tests__/sqlQuery.test.ts +++ b/ui/src/api/__tests__/sqlQuery.test.ts @@ -24,7 +24,9 @@ import { } from "../../types/errors"; import { listSqlQueryLibraries, + listStoredLibraries, SQL_QUERY_LIBRARY_TYPE_FILTER, + SQL_VIEW_LIBRARY_TYPE_FILTER, sqlQueryExportDownload, sqlQueryExportKickOff, sqlQueryRun, @@ -373,6 +375,88 @@ describe("listSqlQueryLibraries", () => { }); }); +describe("SQL_VIEW_LIBRARY_TYPE_FILTER", () => { + // The SQLView filter shares the SQL on FHIR Library code system with the + // SQLQuery filter; only the type code differs. + it("is the SQL on FHIR Library code system scoped to sql-view", () => { + expect(SQL_VIEW_LIBRARY_TYPE_FILTER).toBe( + "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes|sql-view", + ); + }); +}); + +describe("listStoredLibraries", () => { + // The generalised list function scopes its search by the requested type + // code, issuing the SQLView token for sql-view requests. + it("queries Library with the sql-view type filter", async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ resourceType: "Bundle", entry: [] }), { + status: 200, + headers: { "Content-Type": "application/fhir+json" }, + }), + ); + + await listStoredLibraries("https://example.com/fhir", { + typeCode: "sql-view", + }); + + const url = decodeURIComponent(mockFetch.mock.calls[0][0] as string); + expect(url).toContain("/Library?"); + expect(url).toContain(`type=${SQL_VIEW_LIBRARY_TYPE_FILTER}`); + }); + + // The same function issues the SQLQuery token for sql-query requests, so + // both Library kinds share one code path. + it("queries Library with the sql-query type filter", async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ resourceType: "Bundle", entry: [] }), { + status: 200, + headers: { "Content-Type": "application/fhir+json" }, + }), + ); + + await listStoredLibraries("https://example.com/fhir", { + typeCode: "sql-query", + }); + + const url = decodeURIComponent(mockFetch.mock.calls[0][0] as string); + expect(url).toContain(`type=${SQL_QUERY_LIBRARY_TYPE_FILTER}`); + }); + + // The Authorization header is forwarded for authenticated servers. + it("attaches a bearer token when an access token is provided", async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ resourceType: "Bundle" }), { + status: 200, + headers: { "Content-Type": "application/fhir+json" }, + }), + ); + + await listStoredLibraries("https://example.com/fhir", { + typeCode: "sql-view", + accessToken: "secret", + }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: "Bearer secret" }), + }), + ); + }); + + // A 401 propagates as UnauthorizedError, consistent with the rest of the + // API layer. + it("throws UnauthorizedError on a 401 response", async () => { + mockFetch.mockResolvedValueOnce( + new Response("Unauthorized", { status: 401 }), + ); + await expect( + listStoredLibraries("https://example.com/fhir", { typeCode: "sql-view" }), + ).rejects.toThrow(UnauthorizedError); + }); +}); + describe("sqlQueryExportKickOff", () => { // The kick-off must request asynchronous processing and return the polling // URL carried by the Content-Location header. diff --git a/ui/src/api/index.ts b/ui/src/api/index.ts index 5bac26cc9e..9a3549f594 100644 --- a/ui/src/api/index.ts +++ b/ui/src/api/index.ts @@ -64,13 +64,16 @@ export type { ViewDefinition } from "./view"; export { sqlQueryRun, listSqlQueryLibraries, + listStoredLibraries, sqlQueryExportKickOff, sqlQueryExportDownload, SQL_QUERY_LIBRARY_TYPE_SYSTEM, SQL_QUERY_LIBRARY_TYPE_FILTER, + SQL_VIEW_LIBRARY_TYPE_FILTER, SQL_QUERY_LIBRARY_PROFILE, } from "./sqlQuery"; export type { + SqlOnFhirLibraryTypeCode, SqlQueryRunOptions, SqlQueryRunStoredOptions, SqlQueryRunInlineOptions, diff --git a/ui/src/api/sqlQuery.ts b/ui/src/api/sqlQuery.ts index e8fc0cf310..815c8d6fce 100644 --- a/ui/src/api/sqlQuery.ts +++ b/ui/src/api/sqlQuery.ts @@ -51,6 +51,27 @@ export const SQL_QUERY_LIBRARY_TYPE_SYSTEM = */ export const SQL_QUERY_LIBRARY_TYPE_FILTER = `${SQL_QUERY_LIBRARY_TYPE_SYSTEM}|sql-query`; +/** + * Token-search filter that scopes a Library search to the SQLView profile. + * + * Shares the code system with {@link SQL_QUERY_LIBRARY_TYPE_FILTER}; only the + * type code differs. + */ +export const SQL_VIEW_LIBRARY_TYPE_FILTER = `${SQL_QUERY_LIBRARY_TYPE_SYSTEM}|sql-view`; + +/** + * SQL on FHIR Library type codes that the UI can list and run. + */ +export type SqlOnFhirLibraryTypeCode = "sql-query" | "sql-view"; + +/** + * Maps each listable Library type code to its token-search filter. + */ +const LIBRARY_TYPE_FILTERS: Record = { + "sql-query": SQL_QUERY_LIBRARY_TYPE_FILTER, + "sql-view": SQL_VIEW_LIBRARY_TYPE_FILTER, +}; + /** * Profile URL applied to inline SQLQuery Library resources. */ @@ -189,27 +210,32 @@ export async function sqlQueryRun( } /** - * Searches the FHIR server for stored SQLQuery Library resources. + * Searches the FHIR server for stored SQL on FHIR Library resources of a + * given type. * * Uses a `type` token search scoped to the SQL on FHIR Library type code - * system and the `sql-query` code, so unrelated Library resources are - * excluded. + * system and the requested code (`sql-query` or `sql-view`), so unrelated + * Library resources are excluded. SQLQueries and SQLViews are both Library + * resources distinguished only by this code, so a single function lists + * either kind. * * @param baseUrl - The FHIR server base URL. - * @param options - Optional auth configuration. + * @param options - The type code to list, plus optional auth configuration. * @returns A FHIR Bundle containing the matched Library resources. * @throws {UnauthorizedError} When the request receives a 401 response. * @throws {Error} For other non-successful responses. * * @example - * const bundle = await listSqlQueryLibraries("https://example.com/fhir"); + * const bundle = await listStoredLibraries("https://example.com/fhir", { + * typeCode: "sql-view", + * }); */ -export async function listSqlQueryLibraries( +export async function listStoredLibraries( baseUrl: string, - options: AuthOptions = {}, + options: { typeCode: SqlOnFhirLibraryTypeCode } & AuthOptions, ): Promise { const url = buildUrl(baseUrl, "/Library", { - type: SQL_QUERY_LIBRARY_TYPE_FILTER, + type: LIBRARY_TYPE_FILTERS[options.typeCode], }); const headers = buildHeaders({ accessToken: options.accessToken }); @@ -218,6 +244,31 @@ export async function listSqlQueryLibraries( return (await response.json()) as Bundle; } +/** + * Searches the FHIR server for stored SQLQuery Library resources. + * + * A thin wrapper over {@link listStoredLibraries} scoped to the `sql-query` + * type code, retained for the existing callers. + * + * @param baseUrl - The FHIR server base URL. + * @param options - Optional auth configuration. + * @returns A FHIR Bundle containing the matched Library resources. + * @throws {UnauthorizedError} When the request receives a 401 response. + * @throws {Error} For other non-successful responses. + * + * @example + * const bundle = await listSqlQueryLibraries("https://example.com/fhir"); + */ +export async function listSqlQueryLibraries( + baseUrl: string, + options: AuthOptions = {}, +): Promise { + return listStoredLibraries(baseUrl, { + typeCode: "sql-query", + accessToken: options.accessToken, + }); +} + /** * Builds the nested `parameters` Parameters resource carrying runtime * bindings, or returns `undefined` if no non-empty bindings exist. diff --git a/ui/src/hooks/__tests__/useSqlViews.test.tsx b/ui/src/hooks/__tests__/useSqlViews.test.tsx new file mode 100644 index 0000000000..0dfbff4c1b --- /dev/null +++ b/ui/src/hooks/__tests__/useSqlViews.test.tsx @@ -0,0 +1,197 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Tests for the useSqlViews hook. + * + * Verifies that the hook fetches stored SQLView Library resources with the + * `sql-view` type code and maps the bundle to summaries via the shared + * mapper, with parameters always empty for a SQLView. + * + * @author John Grimes + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// Mock the API module so the hook's network call is observable. +vi.mock("../../api", () => ({ + listStoredLibraries: vi.fn(), +})); + +// Mock the config module. +vi.mock("../../config", () => ({ + config: { + fhirBaseUrl: "http://localhost:8080/fhir", + }, +})); + +// Mock the AuthContext. +vi.mock("../../contexts/AuthContext", () => ({ + useAuth: vi.fn(() => ({ + client: { + state: { + tokenResponse: { + access_token: "test-token", + }, + }, + }, + })), +})); + +import { listStoredLibraries } from "../../api"; +import { config } from "../../config"; +import { useSqlViews } from "../useSqlViews"; + +import type { Bundle } from "fhir/r4"; +import type { ReactNode } from "react"; + +/** + * Creates a wrapper component that provides TanStack Query context. + * + * @returns A wrapper function suitable for renderHook. + */ +function createWrapper() { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + }, + }, + }); + return function Wrapper({ children }: { children: ReactNode }) { + return {children}; + }; +} + +// A stored SQLView: Base64-encoded SQL, one dependency, no parameters. +const mockSqlViewBundle: Bundle = { + resourceType: "Bundle", + type: "searchset", + total: 1, + entry: [ + { + resource: { + resourceType: "Library", + id: "view-active-patients", + title: "Active patients", + status: "active", + type: { + coding: [ + { + system: "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes", + code: "sql-view", + }, + ], + }, + content: [{ contentType: "application/sql", data: "U0VMRUNUIDI=" }], + relatedArtifact: [ + { + type: "depends-on", + label: "patients", + resource: "ViewDefinition/patient-demographics", + }, + ], + }, + }, + ], +} as unknown as Bundle; + +describe("useSqlViews", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + // The hook scopes its Library search to the sql-view type code. + it("fetches stored libraries with the sql-view type code", async () => { + vi.mocked(listStoredLibraries).mockResolvedValue(mockSqlViewBundle); + + const { result } = renderHook(() => useSqlViews(), { + wrapper: createWrapper(), + }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(listStoredLibraries).toHaveBeenCalledWith("http://localhost:8080/fhir", { + typeCode: "sql-view", + accessToken: "test-token", + }); + }); + + // The bundle is mapped to summaries via the shared mapper, decoding the + // SQL and surfacing the dependency; parameters are empty for a SQLView. + it("maps the bundle to summaries with empty parameters", async () => { + vi.mocked(listStoredLibraries).mockResolvedValue(mockSqlViewBundle); + + const { result } = renderHook(() => useSqlViews(), { + wrapper: createWrapper(), + }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data).toHaveLength(1); + const summary = result.current.data?.[0]; + expect(summary?.id).toBe("view-active-patients"); + expect(summary?.title).toBe("Active patients"); + expect(summary?.sql).toBe("SELECT 2"); + expect(summary?.relatedArtifacts).toEqual([ + { label: "patients", reference: "ViewDefinition/patient-demographics" }, + ]); + expect(summary?.parameters).toEqual([]); + }); + + // The query is disabled when no FHIR base URL is configured. + it("does not fetch when fhirBaseUrl is not configured", async () => { + vi.mocked(config).fhirBaseUrl = undefined as unknown as string; + vi.mocked(listStoredLibraries).mockResolvedValue(mockSqlViewBundle); + + const { result } = renderHook(() => useSqlViews(), { + wrapper: createWrapper(), + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(result.current.fetchStatus).toBe("idle"); + expect(listStoredLibraries).not.toHaveBeenCalled(); + + // Restore config for other tests. + vi.mocked(config).fhirBaseUrl = "http://localhost:8080/fhir"; + }); + + // The enabled option gates the query, matching useSqlQueryLibraries. + it("does not fetch when enabled is false", async () => { + vi.mocked(listStoredLibraries).mockResolvedValue(mockSqlViewBundle); + + const { result } = renderHook(() => useSqlViews({ enabled: false }), { + wrapper: createWrapper(), + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(result.current.fetchStatus).toBe("idle"); + expect(listStoredLibraries).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/src/hooks/index.ts b/ui/src/hooks/index.ts index 0fa1579774..887640f3c5 100644 --- a/ui/src/hooks/index.ts +++ b/ui/src/hooks/index.ts @@ -62,6 +62,8 @@ export { useSaveViewDefinition } from "./useSaveViewDefinition"; // SQL query operations. export { useSqlQueryLibraries } from "./useSqlQueryLibraries"; +export { useSqlViews } from "./useSqlViews"; +export type { UseSqlViewsOptions, UseSqlViewsResult } from "./useSqlViews"; export { useSaveSqlQueryLibrary } from "./useSaveSqlQueryLibrary"; export { useSqlQueryRun } from "./useSqlQueryRun"; export type { diff --git a/ui/src/hooks/useSqlQueryLibraries.ts b/ui/src/hooks/useSqlQueryLibraries.ts index 547652179b..2043b85680 100644 --- a/ui/src/hooks/useSqlQueryLibraries.ts +++ b/ui/src/hooks/useSqlQueryLibraries.ts @@ -17,7 +17,7 @@ import { useQuery } from "@tanstack/react-query"; -import { listSqlQueryLibraries } from "../api"; +import { listStoredLibraries } from "../api"; import { config } from "../config"; import { mapLibraryBundle } from "./sqlQueryHelpers"; import { useAuth } from "../contexts/AuthContext"; @@ -63,7 +63,8 @@ export function useSqlQueryLibraries( return useQuery({ queryKey: SQL_QUERY_LIBRARIES_QUERY_KEY, queryFn: async () => { - const bundle = await listSqlQueryLibraries(fhirBaseUrl!, { + const bundle = await listStoredLibraries(fhirBaseUrl!, { + typeCode: "sql-query", accessToken, }); return mapLibraryBundle(bundle); diff --git a/ui/src/hooks/useSqlViews.ts b/ui/src/hooks/useSqlViews.ts new file mode 100644 index 0000000000..c0c24be876 --- /dev/null +++ b/ui/src/hooks/useSqlViews.ts @@ -0,0 +1,72 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useQuery } from "@tanstack/react-query"; + +import { listStoredLibraries } from "../api"; +import { config } from "../config"; +import { mapLibraryBundle } from "./sqlQueryHelpers"; +import { useAuth } from "../contexts/AuthContext"; + +import type { SqlQueryLibrarySummary } from "../types/sqlQuery"; +import type { UseQueryResult } from "@tanstack/react-query"; + +/** + * TanStack Query key for the stored SQLView list. Kept distinct from the + * SQLQuery list so each caches and invalidates independently. + */ +export const SQL_VIEWS_QUERY_KEY = ["sqlViews"] as const; + +/** + * Options for {@link useSqlViews}. + */ +export interface UseSqlViewsOptions { + /** Whether to enable the query. Defaults to `true`. */ + enabled?: boolean; +} + +/** + * Hook result type for {@link useSqlViews}. + */ +export type UseSqlViewsResult = UseQueryResult; + +/** + * Fetches stored SQLView Library resources from the FHIR server. + * + * Shares the decoded-summary mapping with {@link useSqlQueryLibraries}; a + * SQLView simply carries no declared parameters. + * + * @param options - Optional query options. + * @returns Query result with summaries of stored SQLView Libraries. + */ +export function useSqlViews(options?: UseSqlViewsOptions): UseSqlViewsResult { + const { fhirBaseUrl } = config; + const { client } = useAuth(); + const accessToken = client?.state.tokenResponse?.access_token; + + return useQuery({ + queryKey: SQL_VIEWS_QUERY_KEY, + queryFn: async () => { + const bundle = await listStoredLibraries(fhirBaseUrl!, { + typeCode: "sql-view", + accessToken, + }); + return mapLibraryBundle(bundle); + }, + enabled: options?.enabled !== false && !!fhirBaseUrl, + }); +} From 606036d06b0209287d9e622c44bfe07e8d4e8380 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:22:14 +1000 Subject: [PATCH 034/162] feat: Surface stored SQLViews in the Select query picker Replace the single Library picker with a grouped picker offering stored SQLQueries and SQLViews, omitting an empty group, and rename the read-only dependency heading from Tables to Views. A selected SQLView runs and exports through the unchanged stored path, resolved as Library/. --- ui/e2e/sqlQuery.spec.ts | 92 ++++++-- ui/src/components/sqlOnFhir/SqlQueryForm.tsx | 14 +- .../sqlOnFhir/SqlQueryStoredTab.tsx | 83 ++++--- .../__tests__/SqlQueryStoredTab.test.tsx | 220 ++++++++++++++++++ 4 files changed, 356 insertions(+), 53 deletions(-) create mode 100644 ui/src/components/sqlOnFhir/__tests__/SqlQueryStoredTab.test.tsx diff --git a/ui/e2e/sqlQuery.spec.ts b/ui/e2e/sqlQuery.spec.ts index 2124e75176..01fdf8a612 100644 --- a/ui/e2e/sqlQuery.spec.ts +++ b/ui/e2e/sqlQuery.spec.ts @@ -26,10 +26,13 @@ import { expect, test } from "@playwright/test"; import { mockCapabilityStatement, mockEmptySqlQueryLibraryBundle, + mockEmptySqlViewLibraryBundle, mockSqlQueryLibrary1, mockSqlQueryLibraryBundle, mockSqlQueryRunCsv, mockSqlQueryRunOperationOutcome, + mockSqlViewLibrary1, + mockSqlViewLibraryBundle, mockViewDefinitionBundle, } from "./fixtures/fhirData"; @@ -52,35 +55,34 @@ async function mockMetadata(page: Page) { /** * Mocks the Library search endpoint, branching on the type filter so the - * SQLQuery search returns the SQLQuery bundle while other Library - * searches return an empty bundle. + * SQLQuery search returns the SQLQuery bundle and the SQLView search returns + * the SQLView bundle, while any other Library search returns an empty bundle. * * @param page - The Playwright Page to attach the route to. - * @param bundle - The Bundle to return for SQLQuery searches. + * @param queryBundle - The Bundle to return for SQLQuery searches. + * @param viewBundle - The Bundle to return for SQLView searches. */ async function mockSqlQueryLibraries( page: Page, - bundle: object = mockSqlQueryLibraryBundle, + queryBundle: object = mockSqlQueryLibraryBundle, + viewBundle: object = mockSqlViewLibraryBundle, ) { await page.route(/\/Library\?[^"]*$/, async (route) => { const url = route.request().url(); - if (url.includes("sql-query")) { - await route.fulfill({ - status: 200, - contentType: "application/fhir+json", - body: JSON.stringify(bundle), - }); - return; - } + const bundle = url.includes("sql-view") + ? viewBundle + : url.includes("sql-query") + ? queryBundle + : { + resourceType: "Bundle", + type: "searchset", + total: 0, + entry: [], + }; await route.fulfill({ status: 200, contentType: "application/fhir+json", - body: JSON.stringify({ - resourceType: "Bundle", - type: "searchset", - total: 0, - entry: [], - }), + body: JSON.stringify(bundle), }); }); } @@ -180,7 +182,7 @@ test.describe("SQL on FHIR page - SQL query mode", () => { await selectSqlQueryMode(page); // Pick the stored library. - await page.getByRole("combobox", { name: /sql query library/i }).click(); + await page.getByRole("combobox", { name: /sql query source/i }).click(); await page .getByRole("option", { name: mockSqlQueryLibrary1.title }) .click(); @@ -203,9 +205,57 @@ test.describe("SQL on FHIR page - SQL query mode", () => { await expect(page.getByRole("cell", { name: "Alice" })).toBeVisible(); }); + test("executes a stored SQLView from the SQL views group", async ({ + page, + }) => { + await mockMetadata(page); + await mockSqlQueryLibraries(page); + await mockViewDefinitions(page); + + // Capture the run request to confirm the SQLView resolves as Library/. + let runBody: { parameter?: Array> } | undefined; + await page.route("**/$sqlquery-run", async (route) => { + runBody = JSON.parse(route.request().postData() ?? "{}"); + await route.fulfill({ + status: 200, + contentType: "text/csv", + body: mockSqlQueryRunCsv, + }); + }); + + await page.goto("/admin/sql-on-fhir"); + await selectSqlQueryMode(page); + + // The picker groups queries and views; pick the SQLView. + await page.getByRole("combobox", { name: /sql query source/i }).click(); + await page.getByRole("option", { name: mockSqlViewLibrary1.title }).click(); + + // The dependency heading reads "Views" rather than "Tables". + await expect(page.getByText("Views", { exact: true })).toBeVisible(); + + await page.getByRole("combobox", { name: /output format/i }).click(); + await page.getByRole("option", { name: "csv" }).click(); + + await page.getByRole("button", { name: /^execute$/i }).click(); + + await expect(page.getByText("2 rows")).toBeVisible(); + await expect(page.getByRole("cell", { name: "Alice" })).toBeVisible(); + + const queryReference = runBody?.parameter?.find( + (p) => p.name === "queryReference", + ); + expect(queryReference?.valueReference).toEqual({ + reference: "Library/view-active-patients", + }); + }); + test("authors and executes an inline Library", async ({ page }) => { await mockMetadata(page); - await mockSqlQueryLibraries(page, mockEmptySqlQueryLibraryBundle); + await mockSqlQueryLibraries( + page, + mockEmptySqlQueryLibraryBundle, + mockEmptySqlViewLibraryBundle, + ); await mockViewDefinitions(page); await mockSqlQueryRunCsvResponse(page); @@ -279,7 +329,7 @@ test.describe("SQL on FHIR page - SQL query mode", () => { await page.goto("/admin/sql-on-fhir"); await selectSqlQueryMode(page); - await page.getByRole("combobox", { name: /sql query library/i }).click(); + await page.getByRole("combobox", { name: /sql query source/i }).click(); await page .getByRole("option", { name: mockSqlQueryLibrary1.title }) .click(); diff --git a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx index e835a7cbb1..74865cd29c 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx @@ -29,7 +29,7 @@ import { PlayIcon, UploadIcon } from "@radix-ui/react-icons"; import { Box, Button, Callout, Card, Flex, Heading, Tabs } from "@radix-ui/themes"; import { useState } from "react"; -import { useSqlQueryLibraries, useViewDefinitions } from "../../hooks"; +import { useSqlQueryLibraries, useSqlViews, useViewDefinitions } from "../../hooks"; import { FieldLabel } from "../FieldLabel"; import { areRuntimeBindingsValid, @@ -106,10 +106,15 @@ export function SqlQueryForm({ const [saveError, setSaveError] = useState(null); const { data: storedLibraries, isLoading: isLoadingLibraries } = useSqlQueryLibraries(); + const { data: storedViews, isLoading: isLoadingViews } = useSqlViews(); const { data: viewDefinitions } = useViewDefinitions(); // Derived: declared parameters surfaced through the runtime bindings panel. - const activeStoredLibrary = storedLibraries?.find((lib) => lib.id === selectedLibraryId); + // The selected source may be a SQLQuery or a SQLView, so locate it across + // both stored lists. + const activeStoredLibrary = [...(storedLibraries ?? []), ...(storedViews ?? [])].find( + (lib) => lib.id === selectedLibraryId, + ); const declaredParameters: Array<{ name: string; type: SqlQueryParameterType; @@ -203,8 +208,9 @@ export function SqlQueryForm({ void; /** Whether the controls should be disabled. */ disabled?: boolean; } /** - * Renders the "Select Library" tab body. + * Renders the "Select query" tab body. * * @param props - The component props. - * @param props.libraries - Stored SQLQuery Library summaries. - * @param props.isLoading - Whether the libraries query is loading. - * @param props.selectedId - The currently selected Library ID. - * @param props.onSelect - Callback fired when the user picks a Library. + * @param props.queries - Stored SQLQuery summaries. + * @param props.views - Stored SQLView summaries. + * @param props.isLoading - Whether either stored list is loading. + * @param props.selectedId - The currently selected logical ID. + * @param props.onSelect - Callback fired when the user picks a source. * @param props.disabled - Whether the controls should be disabled. * @returns The tab body. */ export function SqlQueryStoredTab({ - libraries, + queries, + views, isLoading, selectedId, onSelect, @@ -76,23 +80,31 @@ export function SqlQueryStoredTab({ }: Readonly) { const copyToClipboard = useClipboard(); - const selectedLibrary = libraries?.find((lib) => lib.id === selectedId); + const hasQueries = (queries?.length ?? 0) > 0; + const hasViews = (views?.length ?? 0) > 0; + + // The selection is by logical id, unique across the Library store, so the + // active summary is located across both stored lists. + const selectedLibrary = [...(queries ?? []), ...(views ?? [])].find( + (lib) => lib.id === selectedId, + ); if (isLoading) { return ( - Loading SQL query libraries... + Loading stored queries and views... ); } - if (!libraries || libraries.length === 0) { + if (!hasQueries && !hasViews) { return ( - No SQL query libraries found on the server. Use the "Provide SQL" tab to author one. + No stored SQL queries or SQL views found on the server. Use the "Provide SQL" tab to author + one. ); } @@ -100,7 +112,7 @@ export function SqlQueryStoredTab({ return ( - SQL query library + SQL query source - {libraries.map((lib) => ( - - {lib.title} - - ))} + {hasQueries && ( + + SQL queries + {queries!.map((lib) => ( + + {lib.title} + + ))} + + )} + {hasViews && ( + + SQL views + {views!.map((lib) => ( + + {lib.title} + + ))} + + )} @@ -154,7 +181,7 @@ export function SqlQueryStoredTab({ - Tables + Views {selectedLibrary.relatedArtifacts.length > 0 ? ( {selectedLibrary.relatedArtifacts.map((ra) => ( @@ -192,7 +219,7 @@ export function SqlQueryStoredTab({ ) : ( - Select a stored SQL query library to preview its SQL and bind runtime parameters. + Select a stored query or view to preview its SQL and bind runtime parameters. )} diff --git a/ui/src/components/sqlOnFhir/__tests__/SqlQueryStoredTab.test.tsx b/ui/src/components/sqlOnFhir/__tests__/SqlQueryStoredTab.test.tsx new file mode 100644 index 0000000000..5060255ea3 --- /dev/null +++ b/ui/src/components/sqlOnFhir/__tests__/SqlQueryStoredTab.test.tsx @@ -0,0 +1,220 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Tests for the SqlQueryStoredTab component. + * + * Verifies the grouped picker (SQL queries and SQL views), omission of an + * empty group, the renamed "Views" dependency heading, the SQL preview on + * selection, and the combined empty-state message. + * + * @author John Grimes + */ + +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { render, screen } from "../../../test/testUtils"; +import { SqlQueryStoredTab } from "../SqlQueryStoredTab"; + +import type { SqlQueryLibrarySummary } from "../../../types/sqlQuery"; + +// The hooks barrel transitively imports main.tsx (via AuthContext), which +// runs createRoot at load. Mock it; the tab only needs useClipboard. +vi.mock("../../../hooks", () => ({ + useClipboard: () => vi.fn(), +})); + +/** + * Builds a minimal stored-Library summary for the picker. + * + * @param overrides - Fields to override on the base summary. + * @returns A summary suitable for the SqlQueryStoredTab props. + */ +function makeSummary(overrides: Partial): SqlQueryLibrarySummary { + return { + id: "id", + title: "Title", + sql: "SELECT 1", + relatedArtifacts: [], + parameters: [], + resource: { + resourceType: "Library", + status: "active", + type: { + coding: [ + { + system: "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes", + code: "sql-query", + }, + ], + }, + content: [{ contentType: "application/sql", data: "U0VMRUNUIDE=" }], + }, + ...overrides, + }; +} + +const QUERY = makeSummary({ + id: "patients-by-condition", + title: "Patients by condition", + sql: "SELECT * FROM patients", + parameters: [{ name: "patient_id", type: "string" }], +}); + +const VIEW = makeSummary({ + id: "active-patients", + title: "Active patients", + sql: "SELECT patient_id FROM patients WHERE active = true", + relatedArtifacts: [{ label: "patients", reference: "ViewDefinition/patient-demographics" }], +}); + +describe("SqlQueryStoredTab", () => { + const onSelect = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + // Both groups appear when each list has members. + it("renders a SQL queries group and a SQL views group", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByRole("combobox")); + + expect(screen.getByText("SQL queries")).toBeInTheDocument(); + expect(screen.getByText("SQL views")).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Patients by condition" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Active patients" })).toBeInTheDocument(); + }); + + // An empty group is omitted entirely so no orphan heading shows. + it("omits the SQL views group when there are no views", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByRole("combobox")); + + expect(screen.getByText("SQL queries")).toBeInTheDocument(); + expect(screen.queryByText("SQL views")).not.toBeInTheDocument(); + }); + + // Selecting an option emits its logical id to the parent. + it("emits the selected logical id", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByRole("combobox")); + await user.click(screen.getByRole("option", { name: "Active patients" })); + + expect(onSelect).toHaveBeenCalledWith("active-patients"); + }); + + // A selected SQLView (located across both arrays) previews its decoded SQL. + it("shows the SQL preview for a selected SQLView", () => { + render( + , + ); + + const preview = screen.getByRole("textbox", { + name: /decoded sql preview/i, + }); + expect(preview).toHaveValue("SELECT patient_id FROM patients WHERE active = true"); + }); + + // The dependency heading is renamed from "Tables" to "Views". + it("labels the dependency heading 'Views'", () => { + render( + , + ); + + expect(screen.getByText("Views")).toBeInTheDocument(); + expect(screen.queryByText("Tables")).not.toBeInTheDocument(); + expect(screen.getByText("ViewDefinition/patient-demographics")).toBeInTheDocument(); + }); + + // With neither stored queries nor views, the empty-state copy reflects both. + it("shows a combined empty state when both lists are empty", () => { + render( + , + ); + + expect(screen.getByText(/no stored sql queries or sql views/i)).toBeInTheDocument(); + }); + + // The loading state is shown while either list is still in flight. + it("shows a loading state", () => { + render( + , + ); + + expect(screen.getByText(/loading/i)).toBeInTheDocument(); + }); +}); From c7aa599d5d41f21f0c38729b714a9c8b0c7d78de Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:24:41 +1000 Subject: [PATCH 035/162] feat: Hide runtime parameter values when no parameters apply On the Select query tab the Runtime parameter values section now appears only when the selected source declares parameters, removing the empty panel for SQLViews and param-less SQLQueries. The Provide SQL tab keeps the section always visible to anchor inline parameter authoring. --- ui/e2e/sqlQuery.spec.ts | 3 + ui/src/components/sqlOnFhir/SqlQueryForm.tsx | 25 ++- .../sqlOnFhir/__tests__/SqlQueryForm.test.tsx | 170 ++++++++++++++++++ 3 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 ui/src/components/sqlOnFhir/__tests__/SqlQueryForm.test.tsx diff --git a/ui/e2e/sqlQuery.spec.ts b/ui/e2e/sqlQuery.spec.ts index 01fdf8a612..d35053bf18 100644 --- a/ui/e2e/sqlQuery.spec.ts +++ b/ui/e2e/sqlQuery.spec.ts @@ -230,6 +230,9 @@ test.describe("SQL on FHIR page - SQL query mode", () => { await page.getByRole("combobox", { name: /sql query source/i }).click(); await page.getByRole("option", { name: mockSqlViewLibrary1.title }).click(); + // A SQLView declares no parameters, so the runtime-params section is absent. + await expect(page.getByText("Runtime parameter values")).toBeHidden(); + // The dependency heading reads "Views" rather than "Tables". await expect(page.getByText("Views", { exact: true })).toBeVisible(); diff --git a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx index 74865cd29c..52ee9fd07b 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx @@ -194,6 +194,11 @@ export function SqlQueryForm({ const canSave = !disabled && !isSaving && source === "inline" && canSaveInlineForm(inlineInput); + // On the "Provide SQL" tab the runtime-params section stays anchored as the + // user declares parameters; on the "Select query" tab it appears only when + // the selected source actually declares parameters (a SQLView never does). + const showRuntimeParams = source === "inline" || declaredParameters.length > 0; + return ( @@ -241,15 +246,17 @@ export function SqlQueryForm({ - - Runtime parameter values - - + {showRuntimeParams && ( + + Runtime parameter values + + + )} ): SqlQueryLibrarySummary { + return { + id: "id", + title: "Title", + sql: "SELECT 1", + relatedArtifacts: [], + parameters: [], + resource: { + resourceType: "Library", + status: "active", + type: { + coding: [ + { + system: "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes", + code: "sql-query", + }, + ], + }, + content: [{ contentType: "application/sql", data: "U0VMRUNUIDE=" }], + }, + ...overrides, + }; +} + +const PARAM_QUERY = makeSummary({ + id: "param-query", + title: "Parameterised query", + parameters: [{ name: "patient_id", type: "string" }], +}); + +const PLAIN_QUERY = makeSummary({ + id: "plain-query", + title: "Plain query", + parameters: [], +}); + +const SQL_VIEW = makeSummary({ + id: "sql-view", + title: "Active patients view", + parameters: [], +}); + +// The hooks barrel transitively imports main.tsx (via AuthContext). Mock it +// with the stored lists the form and its picker consume. +vi.mock("../../../hooks", () => ({ + useSqlQueryLibraries: () => ({ + data: [PARAM_QUERY, PLAIN_QUERY], + isLoading: false, + }), + useSqlViews: () => ({ data: [SQL_VIEW], isLoading: false }), + useViewDefinitions: () => ({ data: [] }), + useClipboard: () => vi.fn(), +})); + +const RUNTIME_SECTION = "Runtime parameter values"; + +/** + * Renders the form with inert callbacks. + * + * @returns The userEvent instance for driving interactions. + */ +function renderForm() { + const user = userEvent.setup(); + render( + , + ); + return user; +} + +/** + * Opens the "Select query" picker and chooses the option with the given name. + * + * @param user - The userEvent instance. + * @param optionName - The visible option label to select. + */ +async function selectSource(user: ReturnType, optionName: string) { + await user.click(screen.getByRole("combobox", { name: /sql query source/i })); + await user.click(screen.getByRole("option", { name: optionName })); +} + +describe("SqlQueryForm runtime parameter visibility", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + // With nothing selected on the stored tab, there is nothing to bind. + it("hides the runtime params section when no source is selected", () => { + renderForm(); + expect(screen.queryByText(RUNTIME_SECTION)).not.toBeInTheDocument(); + }); + + // A selected parameterised query exposes its bindings. + it("shows the runtime params section for a parameterised query", async () => { + const user = renderForm(); + await selectSource(user, "Parameterised query"); + expect(screen.getByText(RUNTIME_SECTION)).toBeInTheDocument(); + }); + + // A selected param-less query has nothing to bind, so the section is hidden. + it("hides the runtime params section for a param-less query", async () => { + const user = renderForm(); + await selectSource(user, "Plain query"); + expect(screen.queryByText(RUNTIME_SECTION)).not.toBeInTheDocument(); + }); + + // A SQLView never declares parameters, so the section is hidden. + it("hides the runtime params section for a SQLView", async () => { + const user = renderForm(); + await selectSource(user, "Active patients view"); + expect(screen.queryByText(RUNTIME_SECTION)).not.toBeInTheDocument(); + }); + + // On the "Provide SQL" tab the section is always shown, preserving the + // inline-authoring behaviour. + it("always shows the runtime params section on the Provide SQL tab", async () => { + const user = renderForm(); + await user.click(screen.getByRole("tab", { name: /provide sql/i })); + expect(screen.getByText(RUNTIME_SECTION)).toBeInTheDocument(); + }); +}); From 4c900d1fe1c723de4106108ddcf3aec3009bb2bf Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:31:19 +1000 Subject: [PATCH 036/162] feat: Reference a SQLView when authoring SQL inline Rename the inline Tables editor to Views and let each row reference a stored ViewDefinition or SQLView through a grouped source selector. A collision-safe composite option value carries the source kind so the assembled query emits ViewDefinition/ or Library/ accordingly. --- ui/src/components/sqlOnFhir/SqlQueryForm.tsx | 4 + .../sqlOnFhir/SqlQueryInlineTab.tsx | 94 +++++++---- .../__tests__/SqlQueryInlineTab.test.tsx | 152 ++++++++++++++++++ .../__tests__/sqlQueryFormHelpers.test.ts | 135 ++++++++++++++-- .../sqlOnFhir/sqlQueryFormHelpers.ts | 60 ++++++- ui/src/types/sqlQuery.ts | 15 +- 6 files changed, 408 insertions(+), 52 deletions(-) create mode 100644 ui/src/components/sqlOnFhir/__tests__/SqlQueryInlineTab.test.tsx diff --git a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx index 52ee9fd07b..82bd1e19f7 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx @@ -235,6 +235,10 @@ export function SqlQueryForm({ id: vd.id, name: vd.name, }))} + sqlViews={(storedViews ?? []).map((view) => ({ + id: view.id, + name: view.title, + }))} disabled={disabled || isExecuting} /> {saveError && ( diff --git a/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx b/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx index 24c4903d88..c64faf27e6 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx @@ -16,8 +16,8 @@ */ /** - * "Provide Library" tab body for the SQL query form: SQL editor, tables - * editor and parameter declarations editor. + * "Provide SQL" tab body for the SQL query form: SQL editor, views editor + * and parameter declarations editor. * * @author John Grimes */ @@ -27,6 +27,7 @@ import { Box, Button, Flex, IconButton, Select, Text, TextArea, TextField } from import { FieldGuidance } from "../FieldGuidance"; import { FieldLabel } from "../FieldLabel"; +import { decodeViewReferenceValue, encodeViewReferenceValue } from "./sqlQueryFormHelpers"; import type { SqlQueryParameterDeclaration, @@ -44,7 +45,7 @@ const PARAMETER_TYPES: SqlQueryParameterType[] = [ "dateTime", ]; -interface ViewDefinitionOption { +interface SourceOption { id: string; name: string; } @@ -58,33 +59,36 @@ interface SqlQueryInlineTabProps { sql: string; /** Callback fired when the SQL changes. */ onSqlChange: (sql: string) => void; - /** Configured tables (related artefacts). */ + /** Configured view rows (related artefacts). */ tables: SqlQueryRelatedArtifact[]; - /** Callback fired when the tables list changes. */ + /** Callback fired when the view rows change. */ onTablesChange: (tables: SqlQueryRelatedArtifact[]) => void; /** Configured declared parameters. */ parameters: SqlQueryParameterDeclaration[]; /** Callback fired when the parameters list changes. */ onParametersChange: (parameters: SqlQueryParameterDeclaration[]) => void; - /** Available stored ViewDefinitions for the table selector. */ - viewDefinitions: ViewDefinitionOption[]; + /** Available stored ViewDefinitions for the source selector. */ + viewDefinitions: SourceOption[]; + /** Available stored SQLViews for the source selector. */ + sqlViews: SourceOption[]; /** Whether the controls should be disabled. */ disabled?: boolean; } /** - * Renders the "Provide Library" tab body. + * Renders the "Provide SQL" tab body. * * @param props - The component props. * @param props.title - Library title (used for `Library.title` on save). * @param props.onTitleChange - Callback fired when the title changes. * @param props.sql - SQL text. * @param props.onSqlChange - Callback fired when the SQL changes. - * @param props.tables - Configured tables (related artefacts). - * @param props.onTablesChange - Callback fired when the tables list changes. + * @param props.tables - Configured view rows (related artefacts). + * @param props.onTablesChange - Callback fired when the view rows change. * @param props.parameters - Configured declared parameters. * @param props.onParametersChange - Callback fired when the parameters list changes. - * @param props.viewDefinitions - Available stored ViewDefinitions for the table selector. + * @param props.viewDefinitions - Available stored ViewDefinitions for the source selector. + * @param props.sqlViews - Available stored SQLViews for the source selector. * @param props.disabled - Whether the controls should be disabled. * @returns The tab body. */ @@ -98,15 +102,18 @@ export function SqlQueryInlineTab({ parameters, onParametersChange, viewDefinitions, + sqlViews, disabled = false, }: Readonly) { + const hasSources = viewDefinitions.length > 0 || sqlViews.length > 0; + const handleAddTable = () => { onTablesChange([ ...tables, { rowId: crypto.randomUUID(), label: "", - viewDefinitionId: "", + referenceId: "", }, ]); }; @@ -176,10 +183,10 @@ export function SqlQueryInlineTab({ - Tables + Views {tables.length === 0 && ( - Add at least one table; each maps a label to a stored ViewDefinition. + Add at least one view; each maps a label to a stored ViewDefinition or SQLView. )} @@ -196,39 +203,58 @@ export function SqlQueryInlineTab({ placeholder="e.g. patients" onChange={(e) => handleUpdateTable(table.rowId, { label: e.target.value })} disabled={disabled} - aria-label={`Label for table ${index + 1}`} + aria-label={`Label for view ${index + 1}`} /> {index === 0 && ( - View definition + Source )} - handleUpdateTable(table.rowId, { - viewDefinitionId: value, - }) + handleUpdateTable(table.rowId, decodeViewReferenceValue(value)) } - disabled={disabled || viewDefinitions.length === 0} + disabled={disabled || !hasSources} > - {viewDefinitions.map((vd) => ( - - {vd.name} - - ))} + {viewDefinitions.length > 0 && ( + + View definitions + {viewDefinitions.map((vd) => ( + + {vd.name} + + ))} + + )} + {sqlViews.length > 0 && ( + + SQL views + {sqlViews.map((sv) => ( + + {sv.name} + + ))} + + )} @@ -236,7 +262,7 @@ export function SqlQueryInlineTab({ size="2" variant="soft" color="gray" - aria-label={`Remove table ${index + 1}`} + aria-label={`Remove view ${index + 1}`} onClick={() => handleRemoveTable(table.rowId)} disabled={disabled} > @@ -248,7 +274,7 @@ export function SqlQueryInlineTab({ diff --git a/ui/src/components/sqlOnFhir/__tests__/SqlQueryInlineTab.test.tsx b/ui/src/components/sqlOnFhir/__tests__/SqlQueryInlineTab.test.tsx new file mode 100644 index 0000000000..3a1461afce --- /dev/null +++ b/ui/src/components/sqlOnFhir/__tests__/SqlQueryInlineTab.test.tsx @@ -0,0 +1,152 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Tests for the SqlQueryInlineTab component. + * + * Verifies the "Views" editor (renamed from "Tables"), the grouped source + * selector offering ViewDefinitions and SQLViews, the discriminated row + * update on selection, and the disabled "nothing to reference" state. + * + * @author John Grimes + */ + +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { render, screen } from "../../../test/testUtils"; +import { SqlQueryInlineTab } from "../SqlQueryInlineTab"; + +import type { SqlQueryRelatedArtifact } from "../../../types/sqlQuery"; + +const VIEW_DEFINITIONS = [{ id: "patient-demographics", name: "Patient Demographics" }]; +const SQL_VIEWS = [{ id: "active-patients", name: "Active patients" }]; + +/** + * Renders the inline tab with sensible defaults and inert callbacks. + * + * @param overrides - Props to override on the defaults. + * @param overrides.tables - The view rows to render. + * @param overrides.viewDefinitions - Available ViewDefinition options. + * @param overrides.sqlViews - Available SQLView options. + * @returns The userEvent instance and the onTablesChange spy. + */ +function renderTab( + overrides: { + tables?: SqlQueryRelatedArtifact[]; + viewDefinitions?: Array<{ id: string; name: string }>; + sqlViews?: Array<{ id: string; name: string }>; + } = {}, +) { + const user = userEvent.setup(); + const onTablesChange = vi.fn(); + render( + , + ); + return { user, onTablesChange }; +} + +/** A single empty view row. */ +const EMPTY_ROW: SqlQueryRelatedArtifact = { + rowId: "r1", + label: "patients", + referenceId: "", +}; + +describe("SqlQueryInlineTab", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + // The editor is titled "Views" rather than "Tables". + it("titles the section 'Views'", () => { + renderTab(); + expect(screen.getByText("Views")).toBeInTheDocument(); + expect(screen.queryByText("Tables")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add view/i })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /add table/i })).toBeNull(); + }); + + // The per-row source selector groups ViewDefinitions and SQLViews. + it("groups ViewDefinitions and SQLViews in the source selector", async () => { + const { user } = renderTab({ tables: [EMPTY_ROW] }); + + await user.click(screen.getByRole("combobox", { name: /source for view 1/i })); + + expect(screen.getByText("View definitions")).toBeInTheDocument(); + expect(screen.getByText("SQL views")).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Patient Demographics" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Active patients" })).toBeInTheDocument(); + }); + + // Selecting a ViewDefinition stamps the row with the view-definition kind. + it("updates the row with a view-definition reference", async () => { + const { user, onTablesChange } = renderTab({ tables: [EMPTY_ROW] }); + + await user.click(screen.getByRole("combobox", { name: /source for view 1/i })); + await user.click(screen.getByRole("option", { name: "Patient Demographics" })); + + expect(onTablesChange).toHaveBeenCalledWith([ + expect.objectContaining({ + rowId: "r1", + referenceType: "view-definition", + referenceId: "patient-demographics", + }), + ]); + }); + + // Selecting a SQLView stamps the row with the sql-view kind. + it("updates the row with a sql-view reference", async () => { + const { user, onTablesChange } = renderTab({ tables: [EMPTY_ROW] }); + + await user.click(screen.getByRole("combobox", { name: /source for view 1/i })); + await user.click(screen.getByRole("option", { name: "Active patients" })); + + expect(onTablesChange).toHaveBeenCalledWith([ + expect.objectContaining({ + rowId: "r1", + referenceType: "sql-view", + referenceId: "active-patients", + }), + ]); + }); + + // With neither ViewDefinitions nor SQLViews, the selector is disabled and + // shows a "nothing to reference" placeholder. + it("disables the selector when there is nothing to reference", () => { + renderTab({ tables: [EMPTY_ROW], viewDefinitions: [], sqlViews: [] }); + + const combobox = screen.getByRole("combobox", { name: /source for view 1/i }); + expect(combobox).toBeDisabled(); + expect(combobox).toHaveTextContent(/nothing to reference/i); + }); +}); diff --git a/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts b/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts index df67cf060b..664ba9dd89 100644 --- a/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts +++ b/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts @@ -24,9 +24,43 @@ import { buildParameterTypes, canExecuteInlineForm, canSaveInlineForm, + decodeViewReferenceValue, + encodeViewReferenceValue, isRuntimeValueValid, } from "../sqlQueryFormHelpers"; +describe("encodeViewReferenceValue / decodeViewReferenceValue", () => { + // The codec round-trips a view-definition reference. + it("round-trips a view-definition reference", () => { + const value = encodeViewReferenceValue("view-definition", "vd-1"); + expect(value).toBe("view-definition:vd-1"); + expect(decodeViewReferenceValue(value)).toEqual({ + referenceType: "view-definition", + referenceId: "vd-1", + }); + }); + + // The codec round-trips a sql-view reference. + it("round-trips a sql-view reference", () => { + const value = encodeViewReferenceValue("sql-view", "lib-1"); + expect(value).toBe("sql-view:lib-1"); + expect(decodeViewReferenceValue(value)).toEqual({ + referenceType: "sql-view", + referenceId: "lib-1", + }); + }); + + // Splitting on the first colon keeps ids that themselves contain colons + // intact, so the two id namespaces never collide. + it("preserves ids that contain colons", () => { + const value = encodeViewReferenceValue("sql-view", "urn:uuid:abc:def"); + expect(decodeViewReferenceValue(value)).toEqual({ + referenceType: "sql-view", + referenceId: "urn:uuid:abc:def", + }); + }); +}); + describe("buildInlineSqlQueryLibrary", () => { // The assembled Library carries the SQL on FHIR profile, the // sql-query type code and the SQL both Base64-encoded and as plain @@ -39,7 +73,8 @@ describe("buildInlineSqlQueryLibrary", () => { { rowId: "r1", label: "patients", - viewDefinitionId: "vd-patients", + referenceType: "view-definition", + referenceId: "vd-patients", }, ], parameters: [{ rowId: "p1", name: "patient_id", type: "string" }], @@ -71,6 +106,42 @@ describe("buildInlineSqlQueryLibrary", () => { ]); }); + // A view row backed by a SQLView emits a Library/ reference, while a + // ViewDefinition row emits ViewDefinition/. + it("emits the correct reference prefix per source kind", () => { + const library = buildInlineSqlQueryLibrary({ + sql: "SELECT 1", + tables: [ + { + rowId: "r1", + label: "patients", + referenceType: "view-definition", + referenceId: "patient-demographics", + }, + { + rowId: "r2", + label: "active", + referenceType: "sql-view", + referenceId: "active-patients", + }, + ], + parameters: [], + }); + + expect(library.relatedArtifact).toEqual([ + { + type: "depends-on", + label: "patients", + resource: "ViewDefinition/patient-demographics", + }, + { + type: "depends-on", + label: "active", + resource: "Library/active-patients", + }, + ]); + }); + // Empty title and url do not introduce empty slots on the resource. it("omits empty title and url", () => { const library = buildInlineSqlQueryLibrary({ @@ -106,15 +177,22 @@ describe("canExecuteInlineForm", () => { expect( canExecuteInlineForm({ sql: " ", - tables: [{ rowId: "r1", label: "patients", viewDefinitionId: "vd1" }], + tables: [ + { + rowId: "r1", + label: "patients", + referenceType: "view-definition", + referenceId: "vd1", + }, + ], parameters: [], }), ).toBe(false); }); - // Zero tables prevents execution because the server requires at least + // Zero views prevents execution because the server requires at least // one related artefact. - it("returns false when there are no tables", () => { + it("returns false when there are no views", () => { expect( canExecuteInlineForm({ sql: "SELECT 1", @@ -124,30 +202,49 @@ describe("canExecuteInlineForm", () => { ).toBe(false); }); - // Tables with empty labels or unselected ViewDefinitions are invalid. - it("returns false when a table row is incomplete", () => { + // A view row with a blank label is incomplete. + it("returns false when a view row has no label", () => { expect( canExecuteInlineForm({ sql: "SELECT 1", - tables: [{ rowId: "r1", label: "", viewDefinitionId: "vd1" }], + tables: [ + { + rowId: "r1", + label: "", + referenceType: "view-definition", + referenceId: "vd1", + }, + ], parameters: [], }), ).toBe(false); + }); + + // A view row with no source picked (no referenceType / referenceId) is + // incomplete. + it("returns false when a view row has no source selected", () => { expect( canExecuteInlineForm({ sql: "SELECT 1", - tables: [{ rowId: "r1", label: "patients", viewDefinitionId: "" }], + tables: [{ rowId: "r1", label: "patients", referenceId: "" }], parameters: [], }), ).toBe(false); }); - // Minimum valid input has SQL and at least one well-formed table. + // Minimum valid input has SQL and at least one well-formed view row. it("returns true for the minimum valid input", () => { expect( canExecuteInlineForm({ sql: "SELECT 1", - tables: [{ rowId: "r1", label: "patients", viewDefinitionId: "vd1" }], + tables: [ + { + rowId: "r1", + label: "patients", + referenceType: "sql-view", + referenceId: "lib1", + }, + ], parameters: [], }), ).toBe(true); @@ -160,7 +257,14 @@ describe("canSaveInlineForm", () => { expect( canSaveInlineForm({ sql: "SELECT 1", - tables: [{ rowId: "r1", label: "patients", viewDefinitionId: "vd1" }], + tables: [ + { + rowId: "r1", + label: "patients", + referenceType: "view-definition", + referenceId: "vd1", + }, + ], parameters: [], }), ).toBe(false); @@ -171,7 +275,14 @@ describe("canSaveInlineForm", () => { canSaveInlineForm({ title: "patients-by-condition", sql: "SELECT 1", - tables: [{ rowId: "r1", label: "patients", viewDefinitionId: "vd1" }], + tables: [ + { + rowId: "r1", + label: "patients", + referenceType: "view-definition", + referenceId: "vd1", + }, + ], parameters: [], }), ).toBe(true); diff --git a/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts b/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts index cf31488de3..05ee989725 100644 --- a/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts +++ b/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts @@ -29,6 +29,7 @@ import { import { encodeSql } from "../../utils"; import type { + SqlOnFhirReferenceType, SqlQueryLibrary, SqlQueryParameterDeclaration, SqlQueryParameterType, @@ -42,6 +43,52 @@ import type { const SQL_TEXT_EXTENSION = "https://sql-on-fhir.org/ig/StructureDefinition/sql-text"; +/** + * Encodes a view-row source into a single Radix `Select` item value. + * + * A ViewDefinition and a SQLView Library may share a logical id, so the + * value carries the kind as a prefix to keep the two id namespaces + * collision-safe within one dropdown. + * + * @param type - The kind of source (`view-definition` or `sql-view`). + * @param id - The logical id of the source. + * @returns The composite `${type}:${id}` value. + * + * @example + * encodeViewReferenceValue("sql-view", "active-patients"); + * // "sql-view:active-patients" + */ +export function encodeViewReferenceValue( + type: SqlOnFhirReferenceType, + id: string, +): string { + return `${type}:${id}`; +} + +/** + * Decodes a composite `Select` item value back into its kind and id. + * + * Splits on the first `:` only, so ids that themselves contain colons (for + * example URN-style ids) are preserved intact. + * + * @param value - A value produced by {@link encodeViewReferenceValue}. + * @returns The decoded `referenceType` and `referenceId`. + * + * @example + * decodeViewReferenceValue("view-definition:vd-1"); + * // { referenceType: "view-definition", referenceId: "vd-1" } + */ +export function decodeViewReferenceValue(value: string): { + referenceType: SqlOnFhirReferenceType; + referenceId: string; +} { + const separator = value.indexOf(":"); + return { + referenceType: value.slice(0, separator) as SqlOnFhirReferenceType, + referenceId: value.slice(separator + 1), + }; +} + /** * Inputs to {@link buildInlineSqlQueryLibrary}. */ @@ -52,7 +99,7 @@ export interface BuildInlineLibraryInput { url?: string; /** Plain-text SQL the user wrote. */ sql: string; - /** Tables (related artefacts) configured by the user. */ + /** View rows (related artefacts) configured by the user. */ tables: SqlQueryRelatedArtifact[]; /** Declared runtime parameters. */ parameters: SqlQueryParameterDeclaration[]; @@ -110,7 +157,10 @@ export function buildInlineSqlQueryLibrary( library.relatedArtifact = input.tables.map((table) => ({ type: "depends-on" as const, label: table.label, - resource: `ViewDefinition/${table.viewDefinitionId}`, + resource: + table.referenceType === "sql-view" + ? `Library/${table.referenceId}` + : `ViewDefinition/${table.referenceId}`, })); } @@ -142,7 +192,11 @@ export function canExecuteInlineForm(input: BuildInlineLibraryInput): boolean { if (input.tables.length === 0) { return false; } - if (input.tables.some((t) => t.label.trim() === "" || !t.viewDefinitionId)) { + if ( + input.tables.some( + (t) => t.label.trim() === "" || !t.referenceType || !t.referenceId, + ) + ) { return false; } return true; diff --git a/ui/src/types/sqlQuery.ts b/ui/src/types/sqlQuery.ts index 56f03a05a1..fd7b4ce59a 100644 --- a/ui/src/types/sqlQuery.ts +++ b/ui/src/types/sqlQuery.ts @@ -46,17 +46,26 @@ export type SqlQueryParameterType = | "date" | "dateTime"; +/** + * The kind of stored source a view row references, deciding the reference + * prefix emitted into `relatedArtifact.resource`. + */ +export type SqlOnFhirReferenceType = "view-definition" | "sql-view"; + /** * A `relatedArtifact` entry on a SQLQuery Library, expressed in form-state - * terms. + * terms. A row binds a SQL table label to a chosen stored source, which may + * be a ViewDefinition (`ViewDefinition/`) or a SQLView (`Library/`). */ export interface SqlQueryRelatedArtifact { /** Stable identifier for use as a React `key`. */ rowId: string; /** Table name referenced by the SQL. */ label: string; - /** ID of the referenced ViewDefinition (becomes `ViewDefinition/`). */ - viewDefinitionId: string; + /** The kind of the chosen source; `undefined` until a source is picked. */ + referenceType?: SqlOnFhirReferenceType; + /** Logical id of the chosen ViewDefinition or SQLView; empty until picked. */ + referenceId: string; } /** From 62fbb54c7a8f58aaf180646e0518484e97edf675 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:39:08 +1000 Subject: [PATCH 037/162] test: Update SQL query export e2e for the renamed source picker The stored query picker label changed from "SQL query library" to "SQL query source" when SQLViews joined it; point the export e2e helper at the new accessible name. --- ui/e2e/sqlQueryExport.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/e2e/sqlQueryExport.spec.ts b/ui/e2e/sqlQueryExport.spec.ts index 720d2c322e..5efeedef92 100644 --- a/ui/e2e/sqlQueryExport.spec.ts +++ b/ui/e2e/sqlQueryExport.spec.ts @@ -148,7 +148,7 @@ async function runStoredQuery(page: Page) { await page.goto("/admin/sql-on-fhir"); await page.getByRole("tab", { name: /^sql query$/i }).click(); - await page.getByRole("combobox", { name: /sql query library/i }).click(); + await page.getByRole("combobox", { name: /sql query source/i }).click(); await page.getByRole("option", { name: mockSqlQueryLibrary1.title }).click(); // Enter a runtime value for the declared parameter. From 8f1a24622714e856ce056a257ff4b20e54cb29c5 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:42:01 +1000 Subject: [PATCH 038/162] test: Point inline e2e selectors at the renamed Views editor The inline editor's add-row, label, and source controls were renamed when SQLView references landed; update the inline-authoring e2e paths to match. --- ui/e2e/sqlQuery.spec.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ui/e2e/sqlQuery.spec.ts b/ui/e2e/sqlQuery.spec.ts index d35053bf18..3f3bd20bfe 100644 --- a/ui/e2e/sqlQuery.spec.ts +++ b/ui/e2e/sqlQuery.spec.ts @@ -271,14 +271,12 @@ test.describe("SQL on FHIR page - SQL query mode", () => { // Author the SQL. await page.getByRole("textbox", { name: /^sql$/i }).fill("SELECT 1"); - // Add a table row and select the first ViewDefinition. - await page.getByRole("button", { name: /add table/i }).click(); + // Add a view row and select the first ViewDefinition. + await page.getByRole("button", { name: /add view/i }).click(); await page - .getByRole("textbox", { name: /label for table 1/i }) + .getByRole("textbox", { name: /label for view 1/i }) .fill("patients"); - await page - .getByRole("combobox", { name: /view definition for table 1/i }) - .click(); + await page.getByRole("combobox", { name: /source for view 1/i }).click(); await page.getByRole("option", { name: "Patient Demographics" }).click(); // Use CSV output so the result rendering is deterministic. @@ -306,13 +304,11 @@ test.describe("SQL on FHIR page - SQL query mode", () => { .getByRole("textbox", { name: /library title/i }) .fill("Inline SQL query"); await page.getByRole("textbox", { name: /^sql$/i }).fill("SELECT 1"); - await page.getByRole("button", { name: /add table/i }).click(); + await page.getByRole("button", { name: /add view/i }).click(); await page - .getByRole("textbox", { name: /label for table 1/i }) + .getByRole("textbox", { name: /label for view 1/i }) .fill("patients"); - await page - .getByRole("combobox", { name: /view definition for table 1/i }) - .click(); + await page.getByRole("combobox", { name: /source for view 1/i }).click(); await page.getByRole("option", { name: "Patient Demographics" }).click(); await page.getByRole("button", { name: /save to server/i }).click(); From 94d75315f835eb5f30eb35cc360dda2e0d5e0db5 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 18:49:43 +1000 Subject: [PATCH 039/162] docs: Note SQLView listing in the SQL query client header The module-level comment described the search endpoint as listing SQLQuery Libraries only; it now lists both SQLQueries and SQLViews. --- ui/src/api/sqlQuery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/api/sqlQuery.ts b/ui/src/api/sqlQuery.ts index 815c8d6fce..39c058fd6d 100644 --- a/ui/src/api/sqlQuery.ts +++ b/ui/src/api/sqlQuery.ts @@ -17,7 +17,7 @@ /** * Client for the SQL on FHIR `$sqlquery-run` operation and the search - * endpoint that lists stored SQLQuery Library resources. + * endpoint that lists stored SQLQuery and SQLView Library resources. * * @author John Grimes */ From c02ce060aab27701c76c334c23e943ed0b598463 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 21:43:53 +1000 Subject: [PATCH 040/162] feat: Retain ViewDefinition url and version through encoding The SQL on FHIR ViewDefinition model is a CanonicalResource, but Pathling's stored encoder model dropped its url and version on ingestion. Retain both as additive nullable elements so a ViewDefinition can be matched by its canonical URL when resolving SQL on FHIR dependency references. --- .../encoders/ViewDefinitionResource.java | 56 +++++++++++++++++++ .../encoders/ViewDefinitionEncodingTest.java | 46 +++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/encoders/src/main/java/au/csiro/pathling/encoders/ViewDefinitionResource.java b/encoders/src/main/java/au/csiro/pathling/encoders/ViewDefinitionResource.java index a424e60b4d..778b9e6dec 100644 --- a/encoders/src/main/java/au/csiro/pathling/encoders/ViewDefinitionResource.java +++ b/encoders/src/main/java/au/csiro/pathling/encoders/ViewDefinitionResource.java @@ -73,6 +73,14 @@ public class ViewDefinitionResource extends DomainResource { @Serial private static final long serialVersionUID = 1909997123685548098L; + @Nullable + @Child(name = "url") + private UriType url; + + @Nullable + @Child(name = "version") + private StringType version; + @Nullable @Getter @Child(name = "name") @@ -100,6 +108,50 @@ public class ViewDefinitionResource extends DomainResource { @Child(name = "constant", max = Child.MAX_UNLIMITED) private List constant; + @Nullable + public String getUrl() { + return url == null ? null : url.getValue(); + } + + @Nullable + public UriType getUrlElement() { + return url; + } + + public boolean hasUrlElement() { + return url != null && !url.isEmpty(); + } + + public void setUrlElement(final UriType url) { + this.url = url; + } + + public void setUrl(final String url) { + this.url = url == null ? null : new UriType(url); + } + + @Nullable + public String getVersion() { + return version == null ? null : version.getValue(); + } + + @Nullable + public StringType getVersionElement() { + return version; + } + + public boolean hasVersionElement() { + return version != null && !version.isEmpty(); + } + + public void setVersionElement(final StringType version) { + this.version = version; + } + + public void setVersion(final String version) { + this.version = version == null ? null : new StringType(version); + } + @Nullable public StringType getNameElement() { return name; @@ -187,6 +239,8 @@ public boolean hasConstant() { public DomainResource copy() { final ViewDefinitionResource copy = new ViewDefinitionResource(); copyValues(copy); + copy.url = url != null ? url.copy() : null; + copy.version = version != null ? version.copy() : null; copy.name = name != null ? name.copy() : null; if (fhirVersion != null) { copy.fhirVersion = new ArrayList<>(); @@ -233,6 +287,8 @@ public String fhirType() { @Override public boolean isEmpty() { return super.isEmpty() + && (url == null || url.isEmpty()) + && (version == null || version.isEmpty()) && (name == null || name.isEmpty()) && (fhirVersion == null || fhirVersion.isEmpty()) && (resource == null || resource.isEmpty()) diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/ViewDefinitionEncodingTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/ViewDefinitionEncodingTest.java index d5c200f885..4342797b2e 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/ViewDefinitionEncodingTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/ViewDefinitionEncodingTest.java @@ -55,6 +55,7 @@ import org.hl7.fhir.r4.model.BooleanType; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.UriType; import org.json.JSONException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -127,6 +128,8 @@ void testSchemaHasBasicFields() { // Verify top-level fields exist. assertTrue(schema.getFieldIndex("id").isDefined()); + assertTrue(schema.getFieldIndex("url").isDefined()); + assertTrue(schema.getFieldIndex("version").isDefined()); assertTrue(schema.getFieldIndex("name").isDefined()); assertTrue(schema.getFieldIndex("resource").isDefined()); assertTrue(schema.getFieldIndex("status").isDefined()); @@ -135,6 +138,49 @@ void testSchemaHasBasicFields() { assertTrue(schema.getFieldIndex("constant").isDefined()); } + @Test + void testUrlAndVersionSurviveRoundTrip() { + // The url and version are required to match dependency references by canonical URL, so they + // must be retained through an encode/decode round-trip. + final ViewDefinitionResource original = createSimpleViewDefinition(); + original.setUrlElement(new UriType("https://example.org/ViewDefinition/patients")); + original.setVersionElement(new org.hl7.fhir.r4.model.StringType("2.0")); + + final ExpressionEncoder encoder = + fhirEncodersL0.of(ViewDefinitionResource.class); + final ExpressionEncoder resolvedEncoder = + EncoderUtils.defaultResolveAndBind(encoder); + + final InternalRow serializedRow = resolvedEncoder.createSerializer().apply(original); + final ViewDefinitionResource decoded = + resolvedEncoder.createDeserializer().apply(serializedRow); + + assertTrue(original.equalsDeep(decoded)); + assertEquals("https://example.org/ViewDefinition/patients", decoded.getUrl()); + assertEquals("2.0", decoded.getVersion()); + } + + @Test + void testDecodeWithoutUrlAndVersionYieldsEmptyValues() { + // A ViewDefinition stored without a url or version (the pre-existing case) must decode cleanly + // with absent url and version, leaving it unmatchable by canonical URL. + final ViewDefinitionResource original = createSimpleViewDefinition(); + + final ExpressionEncoder encoder = + fhirEncodersL0.of(ViewDefinitionResource.class); + final ExpressionEncoder resolvedEncoder = + EncoderUtils.defaultResolveAndBind(encoder); + + final InternalRow serializedRow = resolvedEncoder.createSerializer().apply(original); + final ViewDefinitionResource decoded = + resolvedEncoder.createDeserializer().apply(serializedRow); + + assertFalse(decoded.hasUrlElement()); + assertFalse(decoded.hasVersionElement()); + assertNull(decoded.getUrl()); + assertNull(decoded.getVersion()); + } + @Test void testSchemaHandlesRecursiveSelectComponent() { // Level 0: should NOT have nested select.select or unionAll. From e68d3c034ead72b5d5595780f54a367c4e7dfafc Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 22:42:27 +1000 Subject: [PATCH 041/162] feat: Resolve SQL on FHIR dependencies by canonical URL A relatedArtifact.resource on a SQLQuery or SQLView is a canonical URL of a ViewDefinition or SQLView, matched against the referenced resource's url rather than decomposed into a logical id. Extract the shared url/version parsing and candidate selection into one helper reused by both the Library and ViewDefinition paths, resolve ViewDefinitions by filtering on the url column, reject non-canonical references at parse time, and reject ambiguous and unresolvable references with named errors. Request-supplied export views are matched and keyed by url, and a url-less supplied view is rejected. Bumps the server's Pathling core dependency to pick up the ViewDefinition url/version retention the resolution relies on. --- server/pom.xml | 2 +- .../sqlquery/CanonicalReference.java | 180 +++++++++ .../sqlquery/LibraryReferenceResolver.java | 94 +++-- .../sqlquery/SqlDependencyResolver.java | 165 ++++----- .../operations/sqlquery/SqlLibraryParser.java | 8 + .../sqlquery/SqlQueryExportRequestParser.java | 25 +- .../operations/sqlquery/ViewResolver.java | 174 +++++---- .../sqlquery/CanonicalReferenceTest.java | 169 +++++++++ .../sqlquery/RequestViewResolutionTest.java | 175 ++++++--- .../sqlquery/SqlDependencyResolverTest.java | 345 +++++++++--------- .../sqlquery/SqlLibraryFixtures.java | 67 ++++ .../sqlquery/SqlLibraryParserTest.java | 70 +++- .../operations/sqlquery/SqlQueryAuthTest.java | 83 +++-- .../sqlquery/SqlQueryExportFormatIT.java | 2 +- .../sqlquery/SqlQueryExportProviderIT.java | 5 +- .../SqlQueryExportTestConfiguration.java | 16 +- .../sqlquery/SqlQueryRunDeltaIT.java | 35 +- .../SqlQueryRunWithViewDefinitionsIT.java | 11 +- ...lQueryViewDefinitionTestConfiguration.java | 14 + .../sqlquery/SqlViewRunProviderIT.java | 22 +- .../sqlquery/SqlViewTestConfiguration.java | 42 ++- .../sqlquery/ViewRegistrationServiceTest.java | 25 ++ .../operations/sqlquery/ViewResolverTest.java | 146 ++++---- 23 files changed, 1302 insertions(+), 573 deletions(-) create mode 100644 server/src/main/java/au/csiro/pathling/operations/sqlquery/CanonicalReference.java create mode 100644 server/src/test/java/au/csiro/pathling/operations/sqlquery/CanonicalReferenceTest.java diff --git a/server/pom.xml b/server/pom.xml index 9a2c24f1f6..567cdf3436 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -36,7 +36,7 @@ 21 UTF-8 1 - 9.7.1 + 9.8.0 8.10.0 3.5.14 4.0.2 diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/CanonicalReference.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/CanonicalReference.java new file mode 100644 index 0000000000..61a5cca697 --- /dev/null +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/CanonicalReference.java @@ -0,0 +1,180 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; + +/** + * A canonical reference parsed into its {@code url} and optional {@code version}, with the + * candidate-selection rule shared by the {@code Library} and {@code ViewDefinition} resolution + * paths. A canonical reference takes the form {@code [url]} or {@code [url]|[version]}; matching is + * always against the candidate resource's {@code url} element, never its logical id. + * + *

    This is the single place the SQL on FHIR resolution chain parses a canonical and selects among + * candidates sharing a url: + * + *

      + *
    • with an explicit version, the caller has already filtered to that exact version, so any + * remaining candidate suffices; + *
    • without a version, the latest active match wins - preferring {@code status = active}, then + * the lexicographically greatest version string (FHIR does not constrain the shape of a + * version, so this is a reasonable proxy for "latest"). + *
    + * + * @author John Grimes + */ +public final class CanonicalReference { + + @Nonnull private final String url; + + @Nullable private final String version; + + private CanonicalReference(@Nonnull final String url, @Nullable final String version) { + this.url = url; + this.version = version; + } + + /** + * Parses a canonical reference of the form {@code [url]} or {@code [url]|[version]}. An empty + * version suffix (a trailing {@code |}) is treated as no version. + * + * @param canonical the canonical reference to parse + * @return the parsed reference + * @throws InvalidRequestException if the url segment is blank + */ + @Nonnull + public static CanonicalReference parse(@Nonnull final String canonical) { + final int pipe = canonical.indexOf('|'); + final String url = pipe >= 0 ? canonical.substring(0, pipe) : canonical; + final String rawVersion = pipe >= 0 ? canonical.substring(pipe + 1) : null; + if (url.isBlank()) { + throw new InvalidRequestException( + "Canonical reference '" + canonical + "' is missing the url segment"); + } + final String version = rawVersion != null && !rawVersion.isBlank() ? rawVersion : null; + return new CanonicalReference(url, version); + } + + /** + * Indicates whether a value is an absolute canonical URL acceptable as a dependency reference: it + * must use the {@code http://}, {@code https://}, or {@code urn:} scheme, may carry at most one + * {@code |version} suffix, and must not carry a fragment ({@code #...}). + * + * @param value the value to test + * @return {@code true} if the value is an absolute canonical URL + */ + public static boolean isCanonical(@Nullable final String value) { + if (value == null || value.isBlank()) { + return false; + } + if (value.indexOf('#') >= 0) { + // Fragments are not supported. + return false; + } + final int pipe = value.indexOf('|'); + if (pipe >= 0 && value.indexOf('|', pipe + 1) >= 0) { + // At most one version suffix is permitted. + return false; + } + final String url = pipe >= 0 ? value.substring(0, pipe) : value; + return url.startsWith("http://") || url.startsWith("https://") || url.startsWith("urn:"); + } + + /** + * Computes the canonical key identifying a resolved resource: its {@code url} plus its {@code + * version} when it has one ({@code url|version}), else the bare {@code url}. A bare-url reference + * and an explicit {@code url|version} reference that resolve to the same stored resource share + * this key, so the resource de-duplicates and materialises once. + * + * @param url the resolved resource's url + * @param version the resolved resource's version, if any + * @return the canonical key + */ + @Nonnull + public static String key(@Nonnull final String url, @Nullable final String version) { + return version != null && !version.isBlank() ? url + "|" + version : url; + } + + /** + * Selects the most appropriate candidate among those already matched on this reference's url. + * When this reference carries a version, the caller has already filtered to that exact version, + * so the first candidate is returned. Otherwise the latest active candidate wins: preferring + * {@code active} status, then the greatest version string. + * + * @param candidates the candidates already filtered to share this reference's url (and version, + * when one was supplied); must be non-empty + * @param isActive predicate identifying a candidate with {@code active} status + * @param versionOf extracts a candidate's version string (may return {@code null}) + * @param the candidate resource type + * @return the selected candidate + * @throws IllegalStateException if no candidates are supplied + */ + @Nonnull + public T select( + @Nonnull final List candidates, + @Nonnull final Predicate isActive, + @Nonnull final Function versionOf) { + if (candidates.isEmpty()) { + throw new IllegalStateException("Cannot select from an empty candidate list"); + } + if (candidates.size() == 1 || version != null) { + return candidates.get(0); + } + return candidates.stream() + .max( + Comparator.comparing(isActive::test) + .thenComparing(candidate -> Objects.toString(versionOf.apply(candidate), ""))) + .orElseThrow(() -> new IllegalStateException("Cannot select from an empty candidate list")); + } + + /** + * Returns the url segment of this canonical reference. + * + * @return the non-blank url + */ + @Nonnull + public String getUrl() { + return url; + } + + /** + * Returns the version segment of this canonical reference, if one was supplied. + * + * @return the version, or {@code null} when none was supplied + */ + @Nullable + public String getVersion() { + return version; + } + + /** + * Indicates whether this canonical reference carries an explicit version. + * + * @return {@code true} if a version was supplied + */ + public boolean hasVersion() { + return version != null; + } +} diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java index 176a81cede..72bf5d880b 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolver.java @@ -28,10 +28,8 @@ import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import java.util.Comparator; import java.util.List; -import java.util.Objects; +import java.util.Optional; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; @@ -159,24 +157,20 @@ private IBaseResource resolveRelative(@Nonnull final String ref) { } /** - * Resolves a canonical reference of the form {@code [url]} or {@code [url]|[version]}. When - * multiple matches exist, prefers an exact {@code url|version} match, then the latest active - * version (by {@code Library.version} string ordering, since FHIR doesn't constrain its shape). + * Resolves a canonical reference of the form {@code [url]} or {@code [url]|[version]}, matching + * against {@code Library.url}. The url/version split and candidate selection (exact {@code + * url|version}, else latest active by status then version string) are delegated to the shared + * {@link CanonicalReference} helper, so a {@code Library} and a {@code ViewDefinition} are + * selected by identical rules. */ @Nonnull private IBaseResource resolveCanonical(@Nonnull final String canonical) { - final int pipe = canonical.indexOf('|'); - final String url = pipe >= 0 ? canonical.substring(0, pipe) : canonical; - final String version = pipe >= 0 ? canonical.substring(pipe + 1) : null; - - if (url.isBlank()) { - throw new InvalidRequestException("queryReference canonical is missing the url segment"); - } + final CanonicalReference reference = CanonicalReference.parse(canonical); final Dataset libraries = dataSource.read(LIBRARY); - Dataset filtered = libraries.filter(libraries.col("url").equalTo(url)); - if (version != null && !version.isBlank()) { - filtered = filtered.filter(functions.col("version").equalTo(version)); + Dataset filtered = libraries.filter(libraries.col("url").equalTo(reference.getUrl())); + if (reference.hasVersion()) { + filtered = filtered.filter(functions.col("version").equalTo(reference.getVersion())); } final ExpressionEncoder encoder = fhirEncoders.of(LIBRARY); @@ -185,30 +179,60 @@ private IBaseResource resolveCanonical(@Nonnull final String canonical) { throw new ResourceNotFoundException( "Library with canonical reference '" + canonical + "' not found"); } - return pickBestCandidate(candidates, version); + return reference.select( + candidates, + candidate -> ((Library) candidate).getStatus() == PublicationStatus.ACTIVE, + candidate -> ((Library) candidate).getVersion()); } /** - * Selects the most appropriate Library when multiple match the canonical url. With a version - * suffix all matches already share that version, so any candidate suffices. Without a version - * suffix, prefer active over draft/retired, then take the lexicographically greatest version - * string (a reasonable proxy for "latest" given FHIR's freeform versioning). + * Attempts to resolve a canonical reference to a stored {@code SQLView} {@code Library} by + * matching {@code Library.url}, returning empty when no Library matches. Used by {@link + * SqlDependencyResolver} for the SQLView arm of canonical dependency resolution, where a + * non-match is not an error in itself (the reference may instead name a {@code ViewDefinition}). + * + *

    The candidate-selection rules match {@link #resolveCanonical}: an exact {@code url|version} + * match, else the latest active version. The {@code Library} metadata READ check is enforced + * (when authorisation is enabled) only once a Library is actually matched, so referencing a URL + * that names a {@code ViewDefinition} the caller can read is not blocked by missing {@code + * Library} authority. + * + * @param canonical the canonical reference to resolve + * @return the resolved Library, or empty if no Library matches the canonical url */ @Nonnull - private IBaseResource pickBestCandidate( - @Nonnull final List candidates, @Nullable final String version) { - if (candidates.size() == 1 || version != null) { - return candidates.get(0); + public Optional tryResolveSqlViewLibrary(@Nonnull final String canonical) { + final CanonicalReference reference = CanonicalReference.parse(canonical); + + final Dataset libraries; + try { + libraries = dataSource.read(LIBRARY); + } catch (final IllegalArgumentException e) { + // The server holds no Library data at all, so the reference simply does not match a SQLView. + if (e.getMessage() != null && e.getMessage().contains("No data found for resource type")) { + return Optional.empty(); + } + throw e; + } + Dataset filtered = libraries.filter(libraries.col("url").equalTo(reference.getUrl())); + if (reference.hasVersion()) { + filtered = filtered.filter(functions.col("version").equalTo(reference.getVersion())); + } + + final ExpressionEncoder encoder = fhirEncoders.of(LIBRARY); + final List candidates = filtered.as(encoder).collectAsList(); + if (candidates.isEmpty()) { + return Optional.empty(); + } + + if (serverConfiguration.getAuth().isEnabled()) { + SecurityAspect.checkHasAuthority(PathlingAuthority.resourceAccess(AccessType.READ, LIBRARY)); } - return candidates.stream() - .map(Library.class::cast) - .max( - Comparator.comparing((Library lib) -> lib.getStatus() == PublicationStatus.ACTIVE) - .thenComparing(lib -> Objects.toString(lib.getVersion(), ""))) - .map(IBaseResource.class::cast) - .orElseThrow( - () -> - new ResourceNotFoundException( - "Library with canonical reference could not be selected")); + final IBaseResource selected = + reference.select( + candidates, + candidate -> ((Library) candidate).getStatus() == PublicationStatus.ACTIVE, + candidate -> ((Library) candidate).getVersion()); + return Optional.of((Library) selected); } } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java index 78b0e365bf..c869d31956 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolver.java @@ -18,9 +18,9 @@ package au.csiro.pathling.operations.sqlquery; import au.csiro.pathling.config.ServerConfiguration; -import au.csiro.pathling.errors.AccessDeniedError; import au.csiro.pathling.views.FhirView; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import jakarta.annotation.Nonnull; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -29,41 +29,41 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Library; -import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** * Resolves the transitive dependency graph of a top-level query (a {@code SQLQuery} or a {@code * SQLView}) into a {@link ResolvedDependencyGraph}, without touching Spark. Each {@code - * relatedArtifact} dependency is disambiguated, fetched, authorised, and parsed; {@code SQLView} + * relatedArtifact} dependency is resolved by canonical URL, authorised, and parsed; {@code SQLView} * dependencies are recursed into so the full graph of virtual tables is resolved. * - *

    Reference disambiguation follows the SQL on FHIR resolution contract: + *

    Reference resolution follows the SQL on FHIR canonical-reference contract: a {@code + * relatedArtifact.resource} is an absolute canonical URL (optionally {@code |version}), matched + * against the candidate resource's {@code url} - never its logical id. For a reference the + * resolver: * - *

      - *
    • {@code ViewDefinition/[id]} resolves a {@code ViewDefinition} by logical id; - *
    • {@code Library/[id]} resolves a {@code SQLView} {@code Library} by logical id; - *
    • a bare canonical resolves a {@code ViewDefinition} first (by the canonical's final path - * segment as id), falling back to a {@code SQLView} {@code Library} by canonical url. - *
    + *
      + *
    1. prefers a request-supplied view whose URL matches; + *
    2. otherwise searches stored {@code ViewDefinition}s by url, then {@code SQLView Library}s by + * url; + *
    3. rejects a URL that matches both a ViewDefinition and a SQLView as ambiguous, and a URL that + * matches neither as not found - each naming the label and the reference. + *
    * - *

    The resolution memoises by canonical key, so a node referenced from more than one place (a - * diamond) is resolved once and shared. A reference encountered while it is already on the - * resolution stack is a cycle and is rejected, as is a graph that nests deeper than the configured - * {@code maxDependencyDepth}. All such failures are reported before any Spark execution. + *

    The resolution memoises by the resolved canonical key (the matched resource's url plus its + * version, else the bare url), so a node referenced from more than one place (a diamond) - + * including a bare-url reference and a {@code url|version} reference to the same stored resource - + * is resolved once and shared. A reference encountered while it is already on the resolution stack + * is a cycle and is rejected, as is a graph that nests deeper than the configured {@code + * maxDependencyDepth}. All such failures are reported before any Spark execution. * * @author John Grimes */ @Component public class SqlDependencyResolver { - private static final String VIEW_DEFINITION_PREFIX = "ViewDefinition/"; - - private static final String LIBRARY_PREFIX = "Library/"; - @Nonnull private final ViewResolver viewResolver; @Nonnull private final LibraryReferenceResolver libraryReferenceResolver; @@ -75,9 +75,8 @@ public class SqlDependencyResolver { /** * Constructs a new SqlDependencyResolver. * - * @param viewResolver resolves ViewDefinition leaves, preferring request-supplied views - * @param libraryReferenceResolver resolves a SQLView Library reference (relative or canonical) - * from storage + * @param viewResolver resolves ViewDefinition leaves by url, preferring request-supplied views + * @param libraryReferenceResolver resolves a SQLView Library by canonical url from storage * @param libraryParser the shared parser for SQLView Libraries * @param serverConfiguration the server configuration (auth toggle and the dependency depth cap) */ @@ -97,12 +96,13 @@ public SqlDependencyResolver( * Resolves the dependency graph for a parsed top-level query. * * @param topLevel the parsed top-level query (SQLQuery or SQLView) - * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy, used - * for the top-level query's direct references; nested SQLView dependencies resolve from - * storage only + * @param suppliedViews request-supplied views keyed by the canonical URL they satisfy, used for + * the top-level query's direct references; nested SQLView dependencies resolve from storage + * only * @return the resolved dependency graph, topologically ordered - * @throws InvalidRequestException if a reference is unresolvable, a cycle or depth-limit breach - * is detected, or a dependency is a malformed or wrong-typed resource + * @throws InvalidRequestException if a reference is ambiguous, a cycle or depth-limit breach is + * detected, or a dependency is a malformed or wrong-typed resource + * @throws ResourceNotFoundException if a reference matches no stored ViewDefinition or SQLView */ @Nonnull public ResolvedDependencyGraph resolve( @@ -138,7 +138,12 @@ private Map resolveReferences( return keysByLabel; } - /** Resolves a single reference into the canonical key of its node, registering it if new. */ + /** + * Resolves a single reference into the canonical key of its node, registering it if new. A + * request-supplied view wins; otherwise the canonical url is matched against stored + * ViewDefinitions then SQLView Libraries, rejecting an ambiguous match (both types) and a + * not-found match (neither type). + */ @Nonnull private String resolveReference( @Nonnull final ViewArtifactReference reference, @@ -158,24 +163,40 @@ private String resolveReference( + "')"); } - final String referenceValue = reference.getCanonicalUrl(); + // A request-supplied view, matched by url, is preferred over storage. + final Optional suppliedView = + viewResolver.resolveSuppliedView(reference, suppliedViews); + if (suppliedView.isPresent()) { + return registerLeaf(suppliedView.get(), nodesByKey); + } + + // Search stored ViewDefinitions, then stored SQLView Libraries, both by url. + final Optional storedViewDefinition = + viewResolver.resolveStoredViewDefinition(reference); + final Optional sqlViewLibrary = + libraryReferenceResolver.tryResolveSqlViewLibrary(reference.getCanonicalUrl()); - if (referenceValue.startsWith(VIEW_DEFINITION_PREFIX)) { - // An explicit ViewDefinition reference is authoritative; no fallback. - return registerLeaf(viewResolver.resolveViewDefinition(reference, suppliedViews), nodesByKey); + if (storedViewDefinition.isPresent() && sqlViewLibrary.isPresent()) { + throw new InvalidRequestException( + "The dependency for label '" + + reference.getLabel() + + "' (reference '" + + reference.getCanonicalUrl() + + "') is ambiguous: the canonical URL matches both a ViewDefinition and a SQLView"); } - if (referenceValue.startsWith(LIBRARY_PREFIX)) { - // An explicit Library reference is authoritative; resolve a SQLView, no fallback. - return resolveSqlView(reference, depth, maxDepth, resolutionStack, nodesByKey); + if (storedViewDefinition.isPresent()) { + return registerLeaf(storedViewDefinition.get(), nodesByKey); } - - // Bare canonical (or bare id): try a ViewDefinition first, then fall back to a SQLView. - final Optional viewDefinition = - viewResolver.tryResolveViewDefinition(reference, suppliedViews); - if (viewDefinition.isPresent()) { - return registerLeaf(viewDefinition.get(), nodesByKey); + if (sqlViewLibrary.isPresent()) { + return resolveSqlView( + sqlViewLibrary.get(), reference, depth, maxDepth, resolutionStack, nodesByKey); } - return resolveSqlView(reference, depth, maxDepth, resolutionStack, nodesByKey); + throw new ResourceNotFoundException( + "Failed to resolve the dependency for label '" + + reference.getLabel() + + "' with reference '" + + reference.getCanonicalUrl() + + "': no ViewDefinition or SQLView matches that canonical URL"); } /** Registers a resolved ViewDefinition leaf (deduplicating diamonds) and returns its key. */ @@ -188,22 +209,21 @@ private String registerLeaf( } /** - * Resolves a reference to a {@code SQLView} {@code Library}, recursing into its own dependencies. - * Detects diamonds (already resolved), cycles (currently on the resolution stack), and rejects a - * {@code sql-query} Library referenced as a dependency. + * Resolves a matched {@code SQLView} {@code Library}, recursing into its own dependencies. Keys + * the node by the resolved canonical (the library's url plus its version, else the bare url), so + * two references to the same stored Library - including a bare-url and a {@code url|version} + * reference - deduplicate. Detects diamonds (already resolved), cycles (currently on the + * resolution stack), and rejects a {@code sql-query} Library referenced as a dependency. */ @Nonnull private String resolveSqlView( + @Nonnull final Library library, @Nonnull final ViewArtifactReference reference, final int depth, final int maxDepth, @Nonnull final Set resolutionStack, @Nonnull final Map nodesByKey) { - // The Library is read from storage by LibraryReferenceResolver, which enforces the Library - // metadata READ check when authorisation is enabled. - final Library library = readSqlViewLibrary(reference); - - final String canonicalKey = libraryCanonicalKey(library, reference); + final String canonicalKey = CanonicalReference.key(library.getUrl(), library.getVersion()); // A node already fully resolved is shared (diamond dedup). if (nodesByKey.containsKey(canonicalKey)) { @@ -243,49 +263,4 @@ private String resolveSqlView( nodesByKey.put(canonicalKey, node); return canonicalKey; } - - /** - * Reads the SQLView Library named by the reference from storage, wrapping any resolution failure - * in an actionable error that names the failing label and reference. - */ - @Nonnull - private Library readSqlViewLibrary(@Nonnull final ViewArtifactReference reference) { - final IBaseResource resource; - try { - resource = libraryReferenceResolver.resolve(new Reference(reference.getCanonicalUrl())); - } catch (final AccessDeniedError e) { - // An authorisation denial must surface as a 403, not be reshaped into an unresolvable- - // reference 400. - throw e; - } catch (final RuntimeException e) { - throw new InvalidRequestException( - "Failed to resolve the dependency for label '" - + reference.getLabel() - + "' with reference '" - + reference.getCanonicalUrl() - + "': " - + e.getMessage()); - } - if (resource instanceof final Library library) { - return library; - } - throw new InvalidRequestException( - "The dependency for label '" - + reference.getLabel() - + "' (reference '" - + reference.getCanonicalUrl() - + "') did not resolve to a Library"); - } - - /** - * Computes the canonical key for a resolved SQLView Library: its type and logical id, so two - * references to the same stored Library deduplicate. Falls back to the reference value when the - * resolved resource carries no logical id. - */ - @Nonnull - private String libraryCanonicalKey( - @Nonnull final Library library, @Nonnull final ViewArtifactReference reference) { - final String id = library.getIdElement() == null ? null : library.getIdElement().getIdPart(); - return LIBRARY_PREFIX + (id != null && !id.isBlank() ? id : reference.getCanonicalUrl()); - } } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java index 6dd8fd4629..fa6c1a67a7 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParser.java @@ -198,6 +198,14 @@ private List extractViewReferences(@Nonnull final Library throw new InvalidRequestException( "Each relatedArtifact in the SQL on FHIR Library must have a resource reference"); } + if (!CanonicalReference.isCanonical(resource)) { + throw new InvalidRequestException( + "SQL on FHIR Library relatedArtifact.resource '" + + resource + + "' is not an absolute canonical URL; a canonical URL (http://, https:// or urn:," + + " optionally suffixed with |version) is required to reference a ViewDefinition or" + + " SQLView"); + } references.add(new ViewArtifactReference(label, resource)); } return references; diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java index 82e7dd57b7..07ab5360f6 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportRequestParser.java @@ -18,6 +18,7 @@ package au.csiro.pathling.operations.sqlquery; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.encoders.ViewDefinitionResource; import au.csiro.pathling.errors.UnsupportedFhirPathFeatureError; import au.csiro.pathling.library.io.source.QueryableDataSource; import au.csiro.pathling.operations.view.ViewExecutionHelper; @@ -225,10 +226,11 @@ private IBaseResource selectLibrary( } /** - * Resolves the {@code view} parts into a map keyed by the ViewDefinition id they satisfy, parsing + * Resolves the {@code view} parts into a map keyed by the canonical url they satisfy, parsing * inline views, reading referenced views, applying the per-resource READ check to stored views, * and semantically validating each supplied view (a malformed view is a 400; a semantically - * invalid one a 422). + * invalid one a 422). A supplied view that carries no url is rejected with a 400, since it cannot + * satisfy a canonical dependency reference. */ @Nonnull private Map resolveSuppliedViews(@Nonnull final Parameters parameters) { @@ -253,6 +255,20 @@ private Map resolveSuppliedViews(@Nonnull final Parameters par final boolean inline = viewResource != null; final IBaseResource resolvedResource = viewExecutionHelper.resolveViewInput(viewResource, viewReference); + + // A supplied view satisfies a dependency reference by its canonical url; one without a url + // can never match a canonical reference, so it is rejected up front rather than silently + // ignored, ahead of the heavier parse and semantic validation. + final String url = + resolvedResource instanceof final ViewDefinitionResource viewDefinition + ? viewDefinition.getUrl() + : null; + if (url == null || url.isBlank()) { + throw new InvalidRequestException( + "A supplied 'view' must carry a url to satisfy a canonical dependency reference, but" + + " the supplied view has none"); + } + final FhirView view = parseViewDefinition(resolvedResource); // A stored ViewDefinition is subject to the per-resource READ check; an inline view carries @@ -264,10 +280,7 @@ private Map resolveSuppliedViews(@Nonnull final Parameters par validateViewSemantically(view); - final String matchKey = resolvedResource.getIdElement().getIdPart(); - if (matchKey != null && !matchKey.isBlank()) { - resolved.put(matchKey, view); - } + resolved.put(url, view); } return resolved; } diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java index 6f8984d3cb..47c6bc0077 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewResolver.java @@ -18,8 +18,9 @@ package au.csiro.pathling.operations.sqlquery; import au.csiro.pathling.config.ServerConfiguration; -import au.csiro.pathling.errors.ResourceNotFoundError; -import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.ViewDefinitionResource; +import au.csiro.pathling.io.source.DataSource; import au.csiro.pathling.security.PathlingAuthority; import au.csiro.pathling.security.ResourceAccess.AccessType; import au.csiro.pathling.security.SecurityAspect; @@ -30,22 +31,27 @@ import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import jakarta.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Optional; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; +import org.apache.spark.sql.functions; import org.hl7.fhir.instance.model.api.IBaseResource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** - * Resolves a single {@link ViewArtifactReference} that targets a {@code ViewDefinition} into a - * {@link ResolvedViewDefinition} leaf node, preferring a request-supplied view over server storage - * and performing the per-projected-resource-type authorisation check along the way. Has no Spark - * dependency; the {@link FhirView} is the parsed view configuration, not yet a {@code Dataset}. + * Resolves a {@link ViewArtifactReference} that targets a {@code ViewDefinition} into a {@link + * ResolvedViewDefinition} leaf node, matching the reference's canonical URL against the stored + * {@code ViewDefinition.url} (the logical id plays no part). A request-supplied view is preferred + * over storage, matched by its URL. Has no Spark execution dependency; the {@link FhirView} is the + * parsed view configuration, not yet a {@code Dataset}. * - *

    Used by {@link SqlDependencyResolver} for the {@code ViewDefinition} leaves of a dependency - * graph, both for explicit {@code ViewDefinition/[id]} references and as the first attempt when - * disambiguating a bare canonical reference (which falls back to a {@code SQLView} when no {@code - * ViewDefinition} matches). + *

    Resolution is split into the supplied-view step and the stored-lookup step so the caller can + * detect an ambiguous reference (a URL matching both a {@code ViewDefinition} and a {@code SQLView + * Library}) before committing to either. * * @author John Grimes */ @@ -57,7 +63,11 @@ public class ViewResolver { private static final String VIEW_DEFINITION = "ViewDefinition"; - @Nonnull private final ReadExecutor readExecutor; + private static final String ACTIVE_STATUS = "active"; + + @Nonnull private final DataSource dataSource; + + @Nonnull private final FhirEncoders fhirEncoders; @Nonnull private final ServerConfiguration serverConfiguration; @@ -68,117 +78,125 @@ public class ViewResolver { /** * Constructs a new ViewResolver. * - * @param readExecutor read executor for stored ViewDefinition resources + * @param dataSource the data source used to match stored ViewDefinitions by url + * @param fhirEncoders FHIR encoders used to decode the matched ViewDefinition rows * @param serverConfiguration server configuration (used for the auth toggle) * @param fhirContext FHIR context used to serialise the resolved ViewDefinition for parsing */ @Autowired public ViewResolver( - @Nonnull final ReadExecutor readExecutor, + @Nonnull final DataSource dataSource, + @Nonnull final FhirEncoders fhirEncoders, @Nonnull final ServerConfiguration serverConfiguration, @Nonnull final FhirContext fhirContext) { - this.readExecutor = readExecutor; + this.dataSource = dataSource; + this.fhirEncoders = fhirEncoders; this.serverConfiguration = serverConfiguration; this.fhirContext = fhirContext; this.gson = ViewDefinitionGson.create(); } /** - * Resolves a {@code ViewDefinition} reference into a {@link ResolvedViewDefinition}, throwing - * when no matching view is supplied or stored. Used for explicit {@code ViewDefinition/[id]} - * references, where the contract forbids falling back to another resource type. + * Resolves a reference against a request-supplied view, preferring it over storage. A supplied + * view satisfies a reference when its URL matches the reference's url. It carries its own + * authorisation as the request payload, so no metadata or projected-resource READ check applies. * * @param reference the dependency reference to resolve - * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy - * @return the resolved leaf node - * @throws InvalidRequestException if the reference cannot be resolved or parsed + * @param suppliedViews request-supplied views keyed by the canonical URL they satisfy + * @return the resolved leaf node, or empty if no supplied view matches the reference's url */ @Nonnull - public ResolvedViewDefinition resolveViewDefinition( + public Optional resolveSuppliedView( @Nonnull final ViewArtifactReference reference, @Nonnull final Map suppliedViews) { - return tryResolveViewDefinition(reference, suppliedViews) - .orElseThrow( - () -> - new InvalidRequestException( - "Failed to resolve ViewDefinition for label '" - + reference.getLabel() - + "' with reference '" - + reference.getCanonicalUrl() - + "'")); + final CanonicalReference canonical = CanonicalReference.parse(reference.getCanonicalUrl()); + final FhirView supplied = suppliedViews.get(canonical.getUrl()); + if (supplied == null) { + return Optional.empty(); + } + // A supplied view carries no version; its identity is its URL. + return Optional.of(new ResolvedViewDefinition(canonical.getUrl(), supplied)); } /** - * Attempts to resolve a {@code ViewDefinition} reference into a {@link ResolvedViewDefinition}, - * returning empty when no request-supplied view satisfies it and no stored ViewDefinition with - * the reference's id exists. Used as the first step of disambiguating a bare canonical reference, - * which falls back to a {@code SQLView} when this returns empty. + * Resolves a reference against a stored {@code ViewDefinition}, matching the reference's url (and + * version, when supplied) against {@code ViewDefinition.url}. Returns empty when no stored + * ViewDefinition matches, so the caller can fall back to a {@code SQLView}. * - *

    A request-supplied view is preferred over storage and carries its own authorisation as part - * of the request payload. A stored view is subject to the per-projected-resource-type READ check - * (and, when authorisation is enabled, the {@code ViewDefinition} metadata READ check). + *

    When a ViewDefinition matches, the {@code ViewDefinition} metadata READ check and the + * per-projected-resource-type READ check are enforced (when authorisation is enabled). * * @param reference the dependency reference to resolve - * @param suppliedViews request-supplied views keyed by the ViewDefinition id they satisfy - * @return the resolved leaf node, or empty if no ViewDefinition matches - * @throws InvalidRequestException if a stored ViewDefinition exists but cannot be parsed + * @return the resolved leaf node, or empty if no stored ViewDefinition matches + * @throws InvalidRequestException if a matching ViewDefinition cannot be parsed */ @Nonnull - public Optional tryResolveViewDefinition( - @Nonnull final ViewArtifactReference reference, - @Nonnull final Map suppliedViews) { - final String viewDefinitionId = extractViewDefinitionId(reference.getCanonicalUrl()); - final String canonicalKey = VIEW_DEFINITION + "/" + viewDefinitionId; - - // Prefer a request-supplied view that satisfies this reference; it carries its own - // authorisation as the request payload and is not read from storage. - final FhirView suppliedView = suppliedViews.get(viewDefinitionId); - if (suppliedView != null) { - return Optional.of(new ResolvedViewDefinition(canonicalKey, suppliedView)); - } - - final Optional stored = readViewDefinition(viewDefinitionId); - if (stored.isEmpty()) { + public Optional resolveStoredViewDefinition( + @Nonnull final ViewArtifactReference reference) { + final CanonicalReference canonical = CanonicalReference.parse(reference.getCanonicalUrl()); + final List matches = matchByUrl(canonical); + if (matches.isEmpty()) { return Optional.empty(); } + final ViewDefinitionResource chosen = + (ViewDefinitionResource) + canonical.select(matches, ViewResolver::isActive, ViewResolver::versionOf); + // The ViewDefinition was read from storage: enforce the metadata READ check, then parse it and // enforce the per-projected-resource READ check. - checkMetadataReadAuthority(VIEW_DEFINITION); - final FhirView view = parseViewDefinition(stored.get()); + checkMetadataReadAuthority(); + final FhirView view = parseViewDefinition(chosen); checkProjectedResourceReadAuthority(view); + + final String canonicalKey = CanonicalReference.key(chosen.getUrl(), chosen.getVersion()); return Optional.of(new ResolvedViewDefinition(canonicalKey, view)); } /** - * Reads a stored ViewDefinition by its logical id, mapping a missing resource to an empty result - * so the caller can fall back to another resolution strategy. + * Matches stored ViewDefinitions by the reference's url (and version, when supplied), returning + * the decoded resources. When the server holds no ViewDefinition data at all, the reference + * simply does not match, so an empty list is returned rather than surfacing the data source's + * missing-type error. */ @Nonnull - private Optional readViewDefinition(@Nonnull final String id) { + private List matchByUrl(@Nonnull final CanonicalReference canonical) { + final Dataset viewDefinitions; try { - return Optional.of(readExecutor.read(VIEW_DEFINITION, id)); - } catch (final ResourceNotFoundError e) { - return Optional.empty(); + viewDefinitions = dataSource.read(VIEW_DEFINITION); } catch (final IllegalArgumentException e) { - if (e.getMessage() != null && e.getMessage().contains("No data found for resource type")) { - return Optional.empty(); + if (isMissingResourceType(e)) { + return List.of(); } throw e; } + Dataset filtered = + viewDefinitions.filter(viewDefinitions.col("url").equalTo(canonical.getUrl())); + if (canonical.hasVersion()) { + filtered = filtered.filter(functions.col("version").equalTo(canonical.getVersion())); + } + final ExpressionEncoder encoder = fhirEncoders.of(VIEW_DEFINITION); + return filtered.as(encoder).collectAsList(); } /** - * Extracts a ViewDefinition id from a canonical URL or relative reference. Supports relative - * references like {@code ViewDefinition/my-id} and bare ids. + * Indicates whether an {@link IllegalArgumentException} signals that the data source holds no + * data for the requested resource type (as opposed to a genuine error). */ - @Nonnull - private String extractViewDefinitionId(@Nonnull final String canonicalUrl) { - if (canonicalUrl.contains("/")) { - final String[] parts = canonicalUrl.split("/"); - return parts[parts.length - 1]; - } - return canonicalUrl; + private static boolean isMissingResourceType(@Nonnull final IllegalArgumentException e) { + return e.getMessage() != null && e.getMessage().contains("No data found for resource type"); + } + + /** Returns whether a decoded ViewDefinition carries {@code active} status. */ + private static boolean isActive(@Nonnull final IBaseResource resource) { + final ViewDefinitionResource view = (ViewDefinitionResource) resource; + return view.getStatusElement() != null + && ACTIVE_STATUS.equals(view.getStatusElement().getValueAsString()); + } + + /** Returns the version of a decoded ViewDefinition, or null when it has none. */ + private static String versionOf(@Nonnull final IBaseResource resource) { + return ((ViewDefinitionResource) resource).getVersion(); } /** Parses a ViewDefinition resource into a FhirView via JSON round-tripping. */ @@ -193,14 +211,14 @@ private FhirView parseViewDefinition(@Nonnull final IBaseResource viewResource) } /** - * Enforces the metadata READ check for a resource resolved from storage, when authorisation is - * enabled. Reading a {@code ViewDefinition} out of the server requires READ authority on the + * Enforces the metadata READ check for a ViewDefinition resolved from storage, when authorisation + * is enabled. Reading a {@code ViewDefinition} out of the server requires READ authority on the * {@code ViewDefinition} type itself, independent of the data the view projects. */ - private void checkMetadataReadAuthority(@Nonnull final String resourceType) { + private void checkMetadataReadAuthority() { if (serverConfiguration.getAuth().isEnabled()) { SecurityAspect.checkHasAuthority( - PathlingAuthority.resourceAccess(AccessType.READ, resourceType)); + PathlingAuthority.resourceAccess(AccessType.READ, VIEW_DEFINITION)); } } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/CanonicalReferenceTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/CanonicalReferenceTest.java new file mode 100644 index 0000000000..33dde3fe76 --- /dev/null +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/CanonicalReferenceTest.java @@ -0,0 +1,169 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package au.csiro.pathling.operations.sqlquery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.Library; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Unit tests for {@link CanonicalReference} covering the url/version split, the canonical-form + * predicate, and candidate selection (exact version, then prefer-active-then-greatest-version). + * + * @author John Grimes + */ +class CanonicalReferenceTest { + + // --------------------------------------------------------------------------- + // Parsing. + // --------------------------------------------------------------------------- + + @Test + void parsesUrlWithoutVersion() { + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V"); + + assertThat(reference.getUrl()).isEqualTo("https://example.org/V"); + assertThat(reference.getVersion()).isNull(); + assertThat(reference.hasVersion()).isFalse(); + } + + @Test + void parsesUrlWithVersion() { + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V|2"); + + assertThat(reference.getUrl()).isEqualTo("https://example.org/V"); + assertThat(reference.getVersion()).isEqualTo("2"); + assertThat(reference.hasVersion()).isTrue(); + } + + @Test + void treatsAnEmptyVersionSuffixAsAbsent() { + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V|"); + + assertThat(reference.getUrl()).isEqualTo("https://example.org/V"); + assertThat(reference.getVersion()).isNull(); + } + + @Test + void rejectsABlankUrlSegment() { + assertThatThrownBy(() -> CanonicalReference.parse("|2")) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("url"); + } + + // --------------------------------------------------------------------------- + // Canonical-form detection. + // --------------------------------------------------------------------------- + + @ParameterizedTest + @ValueSource( + strings = { + "https://example.org/V", + "http://example.org/V", + "https://example.org/V|2", + "urn:uuid:53fefa32-fcbb-4ff8-8a92-55ee120877b7" + }) + void recognisesAbsoluteCanonicalUrls(final String value) { + assertThat(CanonicalReference.isCanonical(value)).isTrue(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "ViewDefinition/abc", + "Library/abc", + "abc", + "patient-view", + "https://example.org/V#fragment", + "https://example.org/V|1|2", + "" + }) + void rejectsNonCanonicalValues(final String value) { + assertThat(CanonicalReference.isCanonical(value)).isFalse(); + } + + // --------------------------------------------------------------------------- + // Candidate selection. + // --------------------------------------------------------------------------- + + @Test + void selectsTheSoleCandidate() { + final Library only = library("1.0", PublicationStatus.RETIRED); + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V"); + + assertThat(reference.select(List.of(only), isActive(), versionOf())).isSameAs(only); + } + + @Test + void withAVersionReturnsAMatchingCandidateWithoutPreferringStatus() { + // The caller has already filtered to the exact version, so any candidate is acceptable; the + // status preference does not apply when a version was explicitly requested. + final Library first = library("2", PublicationStatus.RETIRED); + final Library second = library("2", PublicationStatus.ACTIVE); + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V|2"); + + assertThat(reference.select(List.of(first, second), isActive(), versionOf())).isSameAs(first); + } + + @Test + void prefersActiveOverRetiredWhenNoVersionSupplied() { + final Library retired = library("3.0", PublicationStatus.RETIRED); + final Library active = library("1.0", PublicationStatus.ACTIVE); + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V"); + + assertThat(reference.select(List.of(retired, active), isActive(), versionOf())) + .isSameAs(active); + } + + @Test + void prefersTheGreatestVersionAmongActiveCandidates() { + final Library v1 = library("1.0", PublicationStatus.ACTIVE); + final Library v2 = library("2.0", PublicationStatus.ACTIVE); + final CanonicalReference reference = CanonicalReference.parse("https://example.org/V"); + + assertThat(reference.select(List.of(v1, v2), isActive(), versionOf())).isSameAs(v2); + } + + // --------------------------------------------------------------------------- + // Helpers. + // --------------------------------------------------------------------------- + + private static Predicate isActive() { + return library -> library.getStatus() == PublicationStatus.ACTIVE; + } + + private static Function versionOf() { + return Library::getVersion; + } + + private static Library library(final String version, final PublicationStatus status) { + final Library library = new Library(); + library.setVersion(version); + library.setStatus(status); + return library; + } +} diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java index ca7fc466b6..e452ab8cca 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/RequestViewResolutionTest.java @@ -18,86 +18,167 @@ package au.csiro.pathling.operations.sqlquery; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import au.csiro.pathling.config.AuthorizationConfiguration; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.encoders.ViewDefinitionResource; import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; -import au.csiro.pathling.read.ReadExecutor; -import au.csiro.pathling.views.FhirView; +import au.csiro.pathling.io.source.DataSource; +import au.csiro.pathling.library.io.source.QueryableDataSource; +import au.csiro.pathling.operations.view.ViewExecutionHelper; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import jakarta.annotation.Nonnull; import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.hl7.fhir.r4.model.CodeType; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; /** - * Unit tests for the request-supplied view resolution path of {@link ViewResolver}: a supplied view - * is matched to a {@code relatedArtifact} reference by id and preferred over server storage, while - * a reference with no matching supplied view falls back to storage. + * Tests for the resolution of request-supplied views against canonical dependency references: a + * supplied view is matched to a reference by its {@code url} and preferred over storage (resolved + * by {@link ViewResolver}), while a supplied view that carries no {@code url} is rejected at parse + * time (by {@link SqlQueryExportRequestParser}). * * @author John Grimes */ class RequestViewResolutionTest { - private ReadExecutor readExecutor; - private ViewResolver resolver; - - @BeforeEach - void setUp() { - readExecutor = mock(ReadExecutor.class); - final ServerConfiguration serverConfiguration = new ServerConfiguration(); - final AuthorizationConfiguration auth = new AuthorizationConfiguration(); - auth.setEnabled(false); - serverConfiguration.setAuth(auth); - resolver = new ViewResolver(readExecutor, serverConfiguration, FhirContext.forR4()); - } + private static final String PATIENTS_URL = "https://example.org/Patients"; + + // --------------------------------------------------------------------------- + // Supplied-view matching by url (ViewResolver). + // --------------------------------------------------------------------------- + + @Nested + class SuppliedViewMatching { + + private ViewResolver resolver; + + @BeforeEach + void setUp() { + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + final AuthorizationConfiguration auth = new AuthorizationConfiguration(); + auth.setEnabled(false); + serverConfiguration.setAuth(auth); + resolver = + new ViewResolver( + mock(DataSource.class), + mock(FhirEncoders.class), + serverConfiguration, + FhirContext.forR4Cached()); + } + + @Test + void suppliedViewIsMatchedByUrlAndPreferredOverStorage() { + final var supplied = + au.csiro.pathling.views.FhirView.ofResource("Patient") + .select( + au.csiro.pathling.views.FhirView.columns( + au.csiro.pathling.views.FhirView.column("id", "id"))) + .build(); + + final Optional resolved = + resolver.resolveSuppliedView( + new ViewArtifactReference("patients", PATIENTS_URL), Map.of(PATIENTS_URL, supplied)); - @Test - void suppliedViewIsPreferredAndStorageIsNotConsulted() { - final FhirView supplied = - FhirView.ofResource("Patient") - .select(FhirView.columns(FhirView.column("id", "id"))) - .build(); - final Map suppliedViews = Map.of("patient-bp", supplied); + assertThat(resolved).isPresent(); + assertThat(resolved.get().getView()).isSameAs(supplied); + assertThat(resolved.get().getCanonicalKey()).isEqualTo(PATIENTS_URL); + } - final ResolvedViewDefinition resolved = - resolver.resolveViewDefinition( - new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), suppliedViews); + @Test + void resolvesNoSuppliedViewWhenNoneMatchesTheUrl() { + final Optional resolved = + resolver.resolveSuppliedView( + new ViewArtifactReference("patients", PATIENTS_URL), Map.of()); - assertThat(resolved.getView()).isSameAs(supplied); - verify(readExecutor, never()).read(eq("ViewDefinition"), anyString()); + assertThat(resolved).isEmpty(); + } } - @Test - void referenceWithNoSuppliedViewFallsBackToStorage() { - when(readExecutor.read("ViewDefinition", "patient-bp")) - .thenReturn(simpleViewDefinition("patient-bp", "Patient")); + // --------------------------------------------------------------------------- + // Url-less supplied-view rejection (SqlQueryExportRequestParser). + // --------------------------------------------------------------------------- - final ResolvedViewDefinition resolved = - resolver.resolveViewDefinition( - new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), Map.of()); + @Nested + class SuppliedViewValidation { - assertThat(resolved.getView().getResource()).isEqualTo("Patient"); - verify(readExecutor).read("ViewDefinition", "patient-bp"); + private ViewExecutionHelper viewExecutionHelper; + private SqlQueryExportRequestParser parser; + + @BeforeEach + void setUp() { + viewExecutionHelper = mock(ViewExecutionHelper.class); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + final AuthorizationConfiguration auth = new AuthorizationConfiguration(); + auth.setEnabled(false); + serverConfiguration.setAuth(auth); + parser = + new SqlQueryExportRequestParser( + mock(SqlQueryPipeline.class), + mock(LibraryReferenceResolver.class), + viewExecutionHelper, + FhirContext.forR4Cached(), + serverConfiguration, + mock(QueryableDataSource.class)); + } + + @Test + void rejectsASuppliedViewWithoutAUrl() { + // The resolved view has no url, so it can never satisfy a canonical reference. + when(viewExecutionHelper.resolveViewInput(any(), any())) + .thenReturn(viewDefinitionWithoutUrl()); + final Parameters body = parametersWithInlineView(); + + assertThatThrownBy( + () -> + parser.parse(requestDetails(body), null, null, null, null, Set.of(), null, null)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("url"); + } + + @Nonnull + private Parameters parametersWithInlineView() { + final Parameters parameters = new Parameters(); + final ParametersParameterComponent view = parameters.addParameter().setName("view"); + view.addPart().setName("viewResource").setResource(viewDefinitionWithoutUrl()); + return parameters; + } + + @Nonnull + private ServletRequestDetails requestDetails(@Nonnull final Parameters body) { + final ServletRequestDetails requestDetails = mock(ServletRequestDetails.class); + when(requestDetails.getResource()).thenReturn(body); + when(requestDetails.getCompleteUrl()).thenReturn("http://localhost/fhir/$sqlquery-export"); + when(requestDetails.getFhirServerBase()).thenReturn("http://localhost/fhir"); + return requestDetails; + } } + // --------------------------------------------------------------------------- + // Helpers. + // --------------------------------------------------------------------------- + @Nonnull - private static ViewDefinitionResource simpleViewDefinition( - @Nonnull final String id, @Nonnull final String resourceType) { + private static ViewDefinitionResource viewDefinitionWithoutUrl() { final ViewDefinitionResource view = new ViewDefinitionResource(); - view.setId(id); - view.setName(new StringType(id + "_view")); - view.setResource(new CodeType(resourceType)); + view.setId("no-url-view"); + view.setName(new StringType("no_url_view")); + view.setResource(new CodeType("Patient")); view.setStatus(new CodeType("active")); final SelectComponent select = new SelectComponent(); final ColumnComponent column = new ColumnComponent(); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java index 4328067c7f..7b13d85076 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlDependencyResolverTest.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,15 +39,18 @@ import org.junit.jupiter.api.Test; /** - * Unit tests for {@link SqlDependencyResolver} covering reference disambiguation, the resolved + * Unit tests for {@link SqlDependencyResolver} covering canonical-URL resolution, the resolved * graph shape for a {@code SQLQuery -> SQLView -> ViewDefinition} chain, request-supplied view - * preference, and the structural rejections (cycles, depth, malformed and wrong-typed - * dependencies). + * preference, diamond de-duplication (including bare-url vs {@code url|version}), and the + * structural rejections (cycles, depth, ambiguity, not-found, and wrong-typed dependencies). * * @author John Grimes */ class SqlDependencyResolverTest { + private static final String PATIENT_VIEW_URL = + SqlLibraryFixtures.viewDefinitionUrl("patient-view"); + private ViewResolver viewResolver; private LibraryReferenceResolver libraryReferenceResolver; private ServerConfiguration serverConfiguration; @@ -69,178 +71,112 @@ void setUp() { } // --------------------------------------------------------------------------- - // Disambiguation. + // Canonical-URL resolution (US1). // --------------------------------------------------------------------------- @Test - void viewDefinitionPrefixResolvesAViewDefinition() { - stubViewDefinition("ViewDefinition/patient-view", "Patient"); - - final ResolvedDependencyGraph graph = - resolver.resolve(sqlQuery("SELECT * FROM p", "p", "ViewDefinition/patient-view"), Map.of()); - - assertThat(graph.getTopLevelKeysByLabel()).containsEntry("p", "ViewDefinition/patient-view"); - assertThat(graph.getNodesByKey().get("ViewDefinition/patient-view")) - .isInstanceOf(ResolvedViewDefinition.class); - } - - @Test - void libraryPrefixResolvesASqlView() { - // Library/base is a SQLView over a ViewDefinition. - stubSqlView("base", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); - stubViewDefinition("ViewDefinition/patient-view", "Patient"); + void resolvesAViewDefinitionByUrl() { + stubStoredViewDefinition(PATIENT_VIEW_URL, PATIENT_VIEW_URL, "Patient"); final ResolvedDependencyGraph graph = - resolver.resolve(sqlQuery("SELECT * FROM b", "b", "Library/base"), Map.of()); + resolver.resolve(sqlQuery("SELECT * FROM p", "p", PATIENT_VIEW_URL), Map.of()); - assertThat(graph.getTopLevelKeysByLabel()).containsEntry("b", "Library/base"); - assertThat(graph.getNodesByKey().get("Library/base")).isInstanceOf(ResolvedSqlView.class); - assertThat(graph.getNodesByKey().get("ViewDefinition/patient-view")) + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("p", PATIENT_VIEW_URL); + assertThat(graph.getNodesByKey().get(PATIENT_VIEW_URL)) .isInstanceOf(ResolvedViewDefinition.class); } @Test - void bareCanonicalResolvesViewDefinitionFirst() { - when(viewResolver.tryResolveViewDefinition( - argThat(ref -> "https://example.org/views/p".equals(ref.getCanonicalUrl())), any())) - .thenReturn( - Optional.of(new ResolvedViewDefinition("ViewDefinition/p", fhirView("Patient")))); + void resolvesASqlViewByUrl() { + final String baseUrl = SqlLibraryFixtures.sqlViewUrl("base"); + stubSqlView(baseUrl, "SELECT * FROM pv", "pv", PATIENT_VIEW_URL); + stubStoredViewDefinition(PATIENT_VIEW_URL, PATIENT_VIEW_URL, "Patient"); final ResolvedDependencyGraph graph = - resolver.resolve(sqlQuery("SELECT * FROM p", "p", "https://example.org/views/p"), Map.of()); + resolver.resolve(sqlQuery("SELECT * FROM b", "b", baseUrl), Map.of()); - assertThat(graph.getTopLevelKeysByLabel()).containsEntry("p", "ViewDefinition/p"); - assertThat(graph.getNodesByKey().get("ViewDefinition/p")) + assertThat(graph.getTopLevelKeysByLabel()).containsEntry("b", baseUrl); + assertThat(graph.getNodesByKey().get(baseUrl)).isInstanceOf(ResolvedSqlView.class); + assertThat(graph.getNodesByKey().get(PATIENT_VIEW_URL)) .isInstanceOf(ResolvedViewDefinition.class); } @Test - void bareCanonicalFallsBackToSqlViewWhenNoViewDefinition() { - final String canonical = "https://example.org/SQLView/active"; - when(viewResolver.tryResolveViewDefinition( - argThat(ref -> canonical.equals(ref.getCanonicalUrl())), any())) - .thenReturn(Optional.empty()); - final Library sqlView = SqlLibraryFixtures.sqlView("SELECT 1"); - sqlView.setId("active"); - sqlView.setUrl(canonical); - when(libraryReferenceResolver.resolve(argThat(ref -> canonical.equals(ref.getReference())))) - .thenReturn(sqlView); + void recursesThroughASqlViewUrlDependencies() { + // SQLQuery -> v1 (SQLView) -> v2 (SQLView) -> ViewDefinition, all by canonical URL. + final String v1Url = SqlLibraryFixtures.sqlViewUrl("v1"); + final String v2Url = SqlLibraryFixtures.sqlViewUrl("v2"); + stubSqlView(v1Url, "SELECT * FROM x", "x", v2Url); + stubSqlView(v2Url, "SELECT * FROM pv", "pv", PATIENT_VIEW_URL); + stubStoredViewDefinition(PATIENT_VIEW_URL, PATIENT_VIEW_URL, "Patient"); final ResolvedDependencyGraph graph = - resolver.resolve(sqlQuery("SELECT * FROM a", "a", canonical), Map.of()); - - assertThat(graph.getTopLevelKeysByLabel()).containsEntry("a", "Library/active"); - assertThat(graph.getNodesByKey().get("Library/active")).isInstanceOf(ResolvedSqlView.class); - } + resolver.resolve(sqlQuery("SELECT * FROM v", "v", v1Url), Map.of()); - @Test - void unresolvableReferenceErrorsNamingLabelAndResource() { - when(libraryReferenceResolver.resolve(any())) - .thenThrow(new ResourceNotFoundException("not found")); - - assertThatThrownBy( - () -> resolver.resolve(sqlQuery("SELECT 1", "x", "Library/missing"), Map.of())) - .isInstanceOf(InvalidRequestException.class) - .hasMessageContaining("x") - .hasMessageContaining("Library/missing"); - } - - @Test - void sqlQueryReferencedAsDependencyIsRejected() { - // A Library/q that is itself a sql-query (not a sql-view) cannot be a dependency. - final Library nested = SqlLibraryFixtures.sqlQuery("SELECT 1"); - nested.setId("q"); - when(libraryReferenceResolver.resolve(argThat(ref -> "Library/q".equals(ref.getReference())))) - .thenReturn(nested); - - assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT 1", "q", "Library/q"), Map.of())) - .isInstanceOf(InvalidRequestException.class) - .hasMessageContaining("sql-query") - .hasMessageContaining("SQLView"); + assertThat(graph.getOrderedNodes()).hasSize(3); + // Dependencies precede dependents: VD, then v2, then v1. + assertThat(graph.getOrderedNodes().get(0).getCanonicalKey()).isEqualTo(PATIENT_VIEW_URL); + assertThat(graph.getOrderedNodes().get(1).getCanonicalKey()).isEqualTo(v2Url); + assertThat(graph.getOrderedNodes().get(2).getCanonicalKey()).isEqualTo(v1Url); } - // --------------------------------------------------------------------------- - // Graph shape. - // --------------------------------------------------------------------------- - @Test void buildsTopologicallyOrderedTwoNodeGraph() { - stubSqlView("base", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); - stubViewDefinition("ViewDefinition/patient-view", "Patient"); + final String baseUrl = SqlLibraryFixtures.sqlViewUrl("base"); + stubSqlView(baseUrl, "SELECT * FROM pv", "pv", PATIENT_VIEW_URL); + stubStoredViewDefinition(PATIENT_VIEW_URL, PATIENT_VIEW_URL, "Patient"); final ResolvedDependencyGraph graph = - resolver.resolve(sqlQuery("SELECT * FROM b", "b", "Library/base"), Map.of()); + resolver.resolve(sqlQuery("SELECT * FROM b", "b", baseUrl), Map.of()); - // The ViewDefinition leaf must appear before the SQLView that depends on it. assertThat(graph.getOrderedNodes()).hasSize(2); - assertThat(graph.getOrderedNodes().get(0).getCanonicalKey()) - .isEqualTo("ViewDefinition/patient-view"); - assertThat(graph.getOrderedNodes().get(1).getCanonicalKey()).isEqualTo("Library/base"); + assertThat(graph.getOrderedNodes().get(0).getCanonicalKey()).isEqualTo(PATIENT_VIEW_URL); + assertThat(graph.getOrderedNodes().get(1).getCanonicalKey()).isEqualTo(baseUrl); - final ResolvedSqlView sqlView = (ResolvedSqlView) graph.getNodesByKey().get("Library/base"); - assertThat(sqlView.getChildKeysByLabel()).containsEntry("pv", "ViewDefinition/patient-view"); - assertThat(graph.getTopLevelKeysByLabel()).containsEntry("b", "Library/base"); + final ResolvedSqlView sqlView = (ResolvedSqlView) graph.getNodesByKey().get(baseUrl); + assertThat(sqlView.getChildKeysByLabel()).containsEntry("pv", PATIENT_VIEW_URL); } @Test - void prefersRequestSuppliedViewDefinitionOverStorage() { - final FhirView supplied = fhirView("Patient"); - when(viewResolver.resolveViewDefinition( - argThat(ref -> "ViewDefinition/patient-view".equals(ref.getCanonicalUrl())), - argThat(map -> map.containsKey("patient-view")))) - .thenReturn(new ResolvedViewDefinition("ViewDefinition/patient-view", supplied)); + void deduplicatesABareUrlAndAVersionedReferenceToTheSameResource() { + // Both the bare url and url|2 resolve to the same stored ViewDefinition (version 2), so they + // normalise to the same canonical key and materialise once. + final String versionedKey = PATIENT_VIEW_URL + "|2"; + stubStoredViewDefinition(PATIENT_VIEW_URL, versionedKey, "Patient"); + stubStoredViewDefinition(PATIENT_VIEW_URL + "|2", versionedKey, "Patient"); final ResolvedDependencyGraph graph = resolver.resolve( - sqlQuery("SELECT * FROM p", "p", "ViewDefinition/patient-view"), - Map.of("patient-view", supplied)); - - final ResolvedViewDefinition node = - (ResolvedViewDefinition) graph.getNodesByKey().get("ViewDefinition/patient-view"); - assertThat(node.getView()).isSameAs(supplied); - } - - // --------------------------------------------------------------------------- - // Nested graphs, diamonds, cycles, depth, and label scoping (US2). - // --------------------------------------------------------------------------- - - @Test - void resolvesAThreeLevelNestedChain() { - // SQLQuery -> v1 (SQLView) -> v2 (SQLView) -> ViewDefinition. - stubSqlView("v1", "SELECT * FROM x", "x", "Library/v2"); - stubSqlView("v2", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); - stubViewDefinition("ViewDefinition/patient-view", "Patient"); - - final ResolvedDependencyGraph graph = - resolver.resolve(sqlQuery("SELECT * FROM v", "v", "Library/v1"), Map.of()); + sqlQueryWithDeps( + "SELECT * FROM a JOIN b", + Map.of("a", PATIENT_VIEW_URL, "b", PATIENT_VIEW_URL + "|2")), + Map.of()); - assertThat(graph.getOrderedNodes()).hasSize(3); - // Dependencies precede dependents: VD, then v2, then v1. - assertThat(graph.getOrderedNodes().get(0).getCanonicalKey()) - .isEqualTo("ViewDefinition/patient-view"); - assertThat(graph.getOrderedNodes().get(1).getCanonicalKey()).isEqualTo("Library/v2"); - assertThat(graph.getOrderedNodes().get(2).getCanonicalKey()).isEqualTo("Library/v1"); + assertThat(graph.getOrderedNodes()).hasSize(1); + assertThat(graph.getNodesByKey()).containsKey(versionedKey); + assertThat(graph.getTopLevelKeysByLabel()) + .containsEntry("a", versionedKey) + .containsEntry("b", versionedKey); } @Test void resolvesADiamondSharedNodeOnce() { - // SQLQuery references both left and right, each of which references the same shared SQLView. - stubSqlView("left", "SELECT * FROM s", "s", "Library/shared"); - stubSqlView("right", "SELECT * FROM s", "s", "Library/shared"); - stubSqlView("shared", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); - stubViewDefinition("ViewDefinition/patient-view", "Patient"); + final String leftUrl = SqlLibraryFixtures.sqlViewUrl("left"); + final String rightUrl = SqlLibraryFixtures.sqlViewUrl("right"); + final String sharedUrl = SqlLibraryFixtures.sqlViewUrl("shared"); + stubSqlView(leftUrl, "SELECT * FROM s", "s", sharedUrl); + stubSqlView(rightUrl, "SELECT * FROM s", "s", sharedUrl); + stubSqlView(sharedUrl, "SELECT * FROM pv", "pv", PATIENT_VIEW_URL); + stubStoredViewDefinition(PATIENT_VIEW_URL, PATIENT_VIEW_URL, "Patient"); final ResolvedDependencyGraph graph = resolver.resolve( - sqlQueryWithDeps( - "SELECT * FROM l JOIN r", Map.of("l", "Library/left", "r", "Library/right")), + sqlQueryWithDeps("SELECT * FROM l JOIN r", Map.of("l", leftUrl, "r", rightUrl)), Map.of()); - // The shared node and the ViewDefinition each appear exactly once. - assertThat(graph.getNodesByKey()).containsKey("Library/shared"); final long sharedCount = graph.getOrderedNodes().stream() - .filter(node -> "Library/shared".equals(node.getCanonicalKey())) + .filter(node -> sharedUrl.equals(node.getCanonicalKey())) .count(); assertThat(sharedCount).isEqualTo(1); assertThat(graph.getOrderedNodes()).hasSize(4); // shared, vd, left, right. @@ -248,41 +184,65 @@ void resolvesADiamondSharedNodeOnce() { @Test void resolvesTheSameLabelInDifferentNodesWithoutCollision() { - // v1 and v2 both use label "t", but for different ViewDefinitions. - stubSqlView("v1", "SELECT * FROM t", "t", "ViewDefinition/a"); - stubSqlView("v2", "SELECT * FROM t", "t", "ViewDefinition/b"); - stubViewDefinition("ViewDefinition/a", "Patient"); - stubViewDefinition("ViewDefinition/b", "Observation"); + final String v1Url = SqlLibraryFixtures.sqlViewUrl("v1"); + final String v2Url = SqlLibraryFixtures.sqlViewUrl("v2"); + final String aUrl = SqlLibraryFixtures.viewDefinitionUrl("a"); + final String bUrl = SqlLibraryFixtures.viewDefinitionUrl("b"); + stubSqlView(v1Url, "SELECT * FROM t", "t", aUrl); + stubSqlView(v2Url, "SELECT * FROM t", "t", bUrl); + stubStoredViewDefinition(aUrl, aUrl, "Patient"); + stubStoredViewDefinition(bUrl, bUrl, "Observation"); final ResolvedDependencyGraph graph = resolver.resolve( - sqlQueryWithDeps( - "SELECT * FROM one, two", Map.of("one", "Library/v1", "two", "Library/v2")), + sqlQueryWithDeps("SELECT * FROM one, two", Map.of("one", v1Url, "two", v2Url)), Map.of()); - final ResolvedSqlView v1 = (ResolvedSqlView) graph.getNodesByKey().get("Library/v1"); - final ResolvedSqlView v2 = (ResolvedSqlView) graph.getNodesByKey().get("Library/v2"); - assertThat(v1.getChildKeysByLabel()).containsEntry("t", "ViewDefinition/a"); - assertThat(v2.getChildKeysByLabel()).containsEntry("t", "ViewDefinition/b"); + final ResolvedSqlView v1 = (ResolvedSqlView) graph.getNodesByKey().get(v1Url); + final ResolvedSqlView v2 = (ResolvedSqlView) graph.getNodesByKey().get(v2Url); + assertThat(v1.getChildKeysByLabel()).containsEntry("t", aUrl); + assertThat(v2.getChildKeysByLabel()).containsEntry("t", bUrl); + } + + @Test + void prefersARequestSuppliedViewOverStorage() { + final FhirView supplied = fhirView("Patient"); + when(viewResolver.resolveSuppliedView( + argThat(ref -> ref != null && PATIENT_VIEW_URL.equals(ref.getCanonicalUrl())), + argThat(map -> map.containsKey(PATIENT_VIEW_URL)))) + .thenReturn(Optional.of(new ResolvedViewDefinition(PATIENT_VIEW_URL, supplied))); + + final ResolvedDependencyGraph graph = + resolver.resolve( + sqlQuery("SELECT * FROM p", "p", PATIENT_VIEW_URL), Map.of(PATIENT_VIEW_URL, supplied)); + + final ResolvedViewDefinition node = + (ResolvedViewDefinition) graph.getNodesByKey().get(PATIENT_VIEW_URL); + assertThat(node.getView()).isSameAs(supplied); } + // --------------------------------------------------------------------------- + // Cycles and depth (keyed by canonical identity). + // --------------------------------------------------------------------------- + @Test void rejectsACycleNamingTheChain() { - stubSqlView("a", "SELECT * FROM b", "b", "Library/b"); - stubSqlView("b", "SELECT * FROM a", "a", "Library/a"); + final String aUrl = SqlLibraryFixtures.sqlViewUrl("a"); + final String bUrl = SqlLibraryFixtures.sqlViewUrl("b"); + stubSqlView(aUrl, "SELECT * FROM b", "b", bUrl); + stubSqlView(bUrl, "SELECT * FROM a", "a", aUrl); - assertThatThrownBy( - () -> resolver.resolve(sqlQuery("SELECT * FROM x", "x", "Library/a"), Map.of())) + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT * FROM x", "x", aUrl), Map.of())) .isInstanceOf(InvalidRequestException.class) - .hasMessageContainingAll("Cyclic", "Library/a", "Library/b"); + .hasMessageContainingAll("Cyclic", aUrl, bUrl); } @Test void rejectsASelfReference() { - stubSqlView("self", "SELECT * FROM s", "s", "Library/self"); + final String selfUrl = SqlLibraryFixtures.sqlViewUrl("self"); + stubSqlView(selfUrl, "SELECT * FROM s", "s", selfUrl); - assertThatThrownBy( - () -> resolver.resolve(sqlQuery("SELECT * FROM x", "x", "Library/self"), Map.of())) + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT * FROM x", "x", selfUrl), Map.of())) .isInstanceOf(InvalidRequestException.class) .hasMessageContaining("Cyclic"); } @@ -290,18 +250,61 @@ void rejectsASelfReference() { @Test void rejectsAGraphDeeperThanTheConfiguredLimit() { serverConfiguration.getSqlQuery().setMaxDependencyDepth(2); - // top -> v1 (depth 1) -> v2 (depth 2) -> v3 (depth 3, exceeds the limit of 2). - stubSqlView("v1", "SELECT * FROM x", "x", "Library/v2"); - stubSqlView("v2", "SELECT * FROM y", "y", "Library/v3"); - stubSqlView("v3", "SELECT * FROM pv", "pv", "ViewDefinition/patient-view"); - stubViewDefinition("ViewDefinition/patient-view", "Patient"); - - assertThatThrownBy( - () -> resolver.resolve(sqlQuery("SELECT * FROM v", "v", "Library/v1"), Map.of())) + final String v1Url = SqlLibraryFixtures.sqlViewUrl("v1"); + final String v2Url = SqlLibraryFixtures.sqlViewUrl("v2"); + final String v3Url = SqlLibraryFixtures.sqlViewUrl("v3"); + stubSqlView(v1Url, "SELECT * FROM x", "x", v2Url); + stubSqlView(v2Url, "SELECT * FROM y", "y", v3Url); + stubSqlView(v3Url, "SELECT * FROM pv", "pv", PATIENT_VIEW_URL); + stubStoredViewDefinition(PATIENT_VIEW_URL, PATIENT_VIEW_URL, "Patient"); + + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT * FROM v", "v", v1Url), Map.of())) .isInstanceOf(InvalidRequestException.class) .hasMessageContainingAll("deeper", "2"); } + // --------------------------------------------------------------------------- + // Error surface (US2): not-found, ambiguity, wrong-typed dependency. + // --------------------------------------------------------------------------- + + @Test + void reportsNotFoundWhenNeitherAViewDefinitionNorASqlViewMatches() { + final String missingUrl = SqlLibraryFixtures.viewDefinitionUrl("missing"); + + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT 1", "x", missingUrl), Map.of())) + .isInstanceOf(ResourceNotFoundException.class) + .hasMessageContaining("x") + .hasMessageContaining(missingUrl); + } + + @Test + void rejectsAnAmbiguousReferenceMatchingBothTypes() { + final String clashUrl = SqlLibraryFixtures.viewDefinitionUrl("clash"); + stubStoredViewDefinition(clashUrl, clashUrl, "Patient"); + stubSqlView(clashUrl, "SELECT 1", "p", PATIENT_VIEW_URL); + + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT 1", "c", clashUrl), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("ambiguous") + .hasMessageContaining("c") + .hasMessageContaining(clashUrl); + } + + @Test + void rejectsASqlQueryReferencedAsADependency() { + // A Library that is itself a sql-query (not a sql-view) cannot be a dependency. + final String queryUrl = SqlLibraryFixtures.sqlViewUrl("q"); + final Library nested = SqlLibraryFixtures.sqlQuery("SELECT 1"); + nested.setUrl(queryUrl); + when(libraryReferenceResolver.tryResolveSqlViewLibrary(queryUrl)) + .thenReturn(Optional.of(nested)); + + assertThatThrownBy(() -> resolver.resolve(sqlQuery("SELECT 1", "q", queryUrl), Map.of())) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("sql-query") + .hasMessageContaining("SQLView"); + } + // --------------------------------------------------------------------------- // Helpers. // --------------------------------------------------------------------------- @@ -323,27 +326,27 @@ private static ParsedSqlQuery sqlQueryWithDeps( return new ParsedSqlQuery(sql, references, List.of(), SqlLibraryParser.SQL_QUERY_TYPE_CODE); } - /** Stubs the view resolver to resolve the given reference to a ViewDefinition over a resource. */ - private void stubViewDefinition( - @Nonnull final String reference, @Nonnull final String resourceType) { - final String key = - reference.startsWith("ViewDefinition/") ? reference : "ViewDefinition/" + reference; - when(viewResolver.resolveViewDefinition( - argThat(ref -> ref != null && reference.equals(ref.getCanonicalUrl())), any())) - .thenReturn(new ResolvedViewDefinition(key, fhirView(resourceType))); + /** + * Stubs the view resolver to resolve a reference whose canonical url matches {@code referenceUrl} + * to a stored ViewDefinition over the given resource type, with the given resolved canonical key. + */ + private void stubStoredViewDefinition( + @Nonnull final String referenceUrl, + @Nonnull final String resolvedKey, + @Nonnull final String resourceType) { + when(viewResolver.resolveStoredViewDefinition( + argThat(ref -> ref != null && referenceUrl.equals(ref.getCanonicalUrl())))) + .thenReturn(Optional.of(new ResolvedViewDefinition(resolvedKey, fhirView(resourceType)))); } - /** Stubs the library resolver to return a stored SQLView with one dependency. */ + /** Stubs the library resolver to return a stored SQLView (carrying {@code url}) with one dep. */ private void stubSqlView( - @Nonnull final String id, + @Nonnull final String url, @Nonnull final String sql, @Nonnull final String depLabel, @Nonnull final String depResource) { - final Library sqlView = SqlLibraryFixtures.sqlView(sql, depLabel, depResource); - sqlView.setId(id); - when(libraryReferenceResolver.resolve( - argThat(ref -> ref != null && ("Library/" + id).equals(ref.getReference())))) - .thenReturn(sqlView); + final Library sqlView = SqlLibraryFixtures.sqlViewWithUrl(url, sql, depLabel, depResource); + when(libraryReferenceResolver.tryResolveSqlViewLibrary(url)).thenReturn(Optional.of(sqlView)); } @Nonnull diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java index f9db0ed393..96321e663f 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryFixtures.java @@ -45,10 +45,77 @@ */ public final class SqlLibraryFixtures { + /** Base canonical URL under which test ViewDefinitions and SQLViews are published. */ + public static final String CANONICAL_BASE = "https://pathling.csiro.au/test/"; + private SqlLibraryFixtures() { // Utility class. } + /** + * Builds the canonical URL for a test ViewDefinition with the given local name. The final segment + * is deliberately the local name so tests can give a ViewDefinition a logical id that differs + * from the URL's final segment. + * + * @param name the local name segment + * @return the canonical URL + */ + @Nonnull + public static String viewDefinitionUrl(@Nonnull final String name) { + return CANONICAL_BASE + "ViewDefinition/" + name; + } + + /** + * Builds the canonical URL for a test SQLView Library with the given local name. + * + * @param name the local name segment + * @return the canonical URL + */ + @Nonnull + public static String sqlViewUrl(@Nonnull final String name) { + return CANONICAL_BASE + "Library/" + name; + } + + /** + * Builds a {@code SQLView} Library that carries its own canonical {@code url} and a single {@code + * depends-on} dependency referenced by canonical URL. + * + * @param url the SQLView's canonical url, against which dependency references resolve it + * @param sql the SQL text to embed + * @param label the dependency table label + * @param resource the dependency resource reference (a canonical URL) + * @return a {@code SQLView} Library carrying a url and one dependency + */ + @Nonnull + public static Library sqlViewWithUrl( + @Nonnull final String url, + @Nonnull final String sql, + @Nonnull final String label, + @Nonnull final String resource) { + final Library library = sqlView(sql, label, resource); + library.setUrl(url); + return library; + } + + /** + * Builds a {@code SQLView} Library that carries its own canonical {@code url} and a set of {@code + * depends-on} dependencies referenced by canonical URL. + * + * @param url the SQLView's canonical url, against which dependency references resolve it + * @param sql the SQL text to embed + * @param dependenciesByLabel the dependencies keyed by table label, each value a canonical URL + * @return a {@code SQLView} Library carrying a url and the given dependencies + */ + @Nonnull + public static Library sqlViewWithUrl( + @Nonnull final String url, + @Nonnull final String sql, + @Nonnull final Map dependenciesByLabel) { + final Library library = sqlView(sql, dependenciesByLabel); + library.setUrl(url); + return library; + } + /** * Builds a {@code SQLQuery} Library carrying the given SQL, with no dependencies or parameters. * diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java index 0528930ef1..4b7ba2a0a6 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlLibraryParserTest.java @@ -33,6 +33,8 @@ import org.hl7.fhir.r4.model.RelatedArtifact.RelatedArtifactType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** Unit tests for {@link SqlLibraryParser} covering both the SQLQuery and SQLView profiles. */ class SqlLibraryParserTest { @@ -58,22 +60,22 @@ void extractsViewReferences() { new RelatedArtifact() .setType(RelatedArtifactType.DEPENDSON) .setLabel("patients") - .setResource("ViewDefinition/patient-view")); + .setResource("https://example.org/ViewDefinition/patient-view")); library.addRelatedArtifact( new RelatedArtifact() .setType(RelatedArtifactType.DEPENDSON) .setLabel("observations") - .setResource("ViewDefinition/obs-view")); + .setResource("https://example.org/ViewDefinition/obs-view")); final ParsedSqlQuery result = parser.parse(library); assertThat(result.getViewReferences()).hasSize(2); assertThat(result.getViewReferences().get(0).getLabel()).isEqualTo("patients"); assertThat(result.getViewReferences().get(0).getCanonicalUrl()) - .isEqualTo("ViewDefinition/patient-view"); + .isEqualTo("https://example.org/ViewDefinition/patient-view"); assertThat(result.getViewReferences().get(1).getLabel()).isEqualTo("observations"); assertThat(result.getViewReferences().get(1).getCanonicalUrl()) - .isEqualTo("ViewDefinition/obs-view"); + .isEqualTo("https://example.org/ViewDefinition/obs-view"); } @Test @@ -112,8 +114,9 @@ void reportsSqlQueryAsNotAView() { @Test void parsesSqlViewWithTypeCodeSqlAndDependencies() { final Library library = SqlLibraryFixtures.sqlView("SELECT * FROM patient_view"); - SqlLibraryFixtures.addDependency(library, "patient_view", "ViewDefinition/patient-view"); - SqlLibraryFixtures.addDependency(library, "base", "Library/base"); + SqlLibraryFixtures.addDependency( + library, "patient_view", "https://example.org/ViewDefinition/patient-view"); + SqlLibraryFixtures.addDependency(library, "base", "https://example.org/Library/base"); final ParsedSqlQuery result = parser.parse(library); @@ -123,8 +126,9 @@ void parsesSqlViewWithTypeCodeSqlAndDependencies() { assertThat(result.getViewReferences()).hasSize(2); assertThat(result.getViewReferences().get(0).getLabel()).isEqualTo("patient_view"); assertThat(result.getViewReferences().get(0).getCanonicalUrl()) - .isEqualTo("ViewDefinition/patient-view"); - assertThat(result.getViewReferences().get(1).getCanonicalUrl()).isEqualTo("Library/base"); + .isEqualTo("https://example.org/ViewDefinition/patient-view"); + assertThat(result.getViewReferences().get(1).getCanonicalUrl()) + .isEqualTo("https://example.org/Library/base"); assertThat(result.getDeclaredParameters()).isEmpty(); } @@ -347,13 +351,61 @@ void acceptsRelatedArtifactLabelWithUnderscoresAndDigits() { new RelatedArtifact() .setType(RelatedArtifactType.DEPENDSON) .setLabel("patients_2024") - .setResource("ViewDefinition/patient-view")); + .setResource("https://example.org/ViewDefinition/patient-view")); final ParsedSqlQuery result = parser.parse(library); assertThat(result.getViewReferences()).hasSize(1); assertThat(result.getViewReferences().get(0).getLabel()).isEqualTo("patients_2024"); } + // --------------------------------------------------------------------------- + // relatedArtifact.resource canonical-form invariant. + // --------------------------------------------------------------------------- + + @ParameterizedTest(name = "rejects non-canonical resource ''{0}''") + @ValueSource( + strings = { + "ViewDefinition/abc", + "Library/abc", + "patient-view", + "https://example.org/V#section" + }) + void rejectsNonCanonicalResourceReference(final String resource) { + final Library library = createMinimalLibrary("SELECT 1"); + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel("t") + .setResource(resource)); + + assertThatThrownBy(() -> parser.parse(library)) + .isInstanceOf(InvalidRequestException.class) + .hasMessageContaining("relatedArtifact.resource") + .hasMessageContaining(resource) + .hasMessageContaining("canonical URL"); + } + + @ParameterizedTest(name = "accepts canonical resource ''{0}''") + @ValueSource( + strings = { + "https://example.org/ViewDefinition/patient-view", + "https://example.org/ViewDefinition/patient-view|2.0", + "http://example.org/Library/base", + "urn:uuid:53fefa32-fcbb-4ff8-8a92-55ee120877b7" + }) + void acceptsCanonicalResourceReference(final String resource) { + final Library library = createMinimalLibrary("SELECT 1"); + library.addRelatedArtifact( + new RelatedArtifact() + .setType(RelatedArtifactType.DEPENDSON) + .setLabel("t") + .setResource(resource)); + + final ParsedSqlQuery result = parser.parse(library); + assertThat(result.getViewReferences()).hasSize(1); + assertThat(result.getViewReferences().get(0).getCanonicalUrl()).isEqualTo(resource); + } + // --------------------------------------------------------------------------- // Parameter profile invariants. // --------------------------------------------------------------------------- diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java index bd15a8b82c..ee46fc0894 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryAuthTest.java @@ -33,19 +33,23 @@ import au.csiro.pathling.errors.AccessDeniedError; import au.csiro.pathling.io.source.DataSource; import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.test.SpringBootUnitTest; import au.csiro.pathling.views.FhirView; import ca.uhn.fhir.context.FhirContext; import jakarta.annotation.Nonnull; import java.util.List; import java.util.Map; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.Library; import org.hl7.fhir.r4.model.Reference; import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.authority.AuthorityUtils; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.oauth2.jwt.Jwt; @@ -54,23 +58,36 @@ /** * Security tests for the {@code $sqlquery-*} resolution path, wiring the real {@link ViewResolver}, * {@link LibraryReferenceResolver}, and {@link SqlDependencyResolver} with authorisation enabled. - * Verifies the metadata-resource authorisation matrix: a stored ViewDefinition dependency requires - * {@code ViewDefinition} READ, a stored SQLView dependency requires {@code Library} READ, the - * per-projected-resource READ still applies at each leaf, and a request-supplied (inline) view - * requires no metadata READ. + * Verifies the metadata-resource authorisation matrix: a stored ViewDefinition dependency (resolved + * by canonical URL) requires {@code ViewDefinition} READ, a stored SQLView dependency requires + * {@code Library} READ, the per-projected-resource READ still applies at each leaf, and a + * request-supplied (inline) view requires no metadata READ. * * @author John Grimes */ -@Tag("UnitTest") +@SpringBootUnitTest class SqlQueryAuthTest { + private static final String PV_URL = "https://example.org/ViewDefinition/pv"; + private static final String BASE_URL = "https://example.org/Library/base"; + + @Autowired private SparkSession spark; + @Autowired private FhirEncoders fhirEncoders; + @Autowired private FhirContext fhirContext; + private ReadExecutor readExecutor; + private DataSource dataSource; private LibraryReferenceResolver libraryReferenceResolver; private SqlDependencyResolver resolver; @BeforeEach void setUp() { readExecutor = mock(ReadExecutor.class); + dataSource = mock(DataSource.class); + // Default to no matches; individual tests stub the datasets they need. + when(dataSource.read("ViewDefinition")).thenReturn(viewDefinitionDataset()); + when(dataSource.read("Library")).thenReturn(libraryDataset()); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); final AuthorizationConfiguration auth = new AuthorizationConfiguration(); auth.setEnabled(true); @@ -78,10 +95,9 @@ void setUp() { serverConfiguration.setSqlQuery(new SqlQueryConfiguration()); final ViewResolver viewResolver = - new ViewResolver(readExecutor, serverConfiguration, FhirContext.forR4Cached()); + new ViewResolver(dataSource, fhirEncoders, serverConfiguration, fhirContext); libraryReferenceResolver = - new LibraryReferenceResolver( - readExecutor, mock(DataSource.class), mock(FhirEncoders.class), serverConfiguration); + new LibraryReferenceResolver(readExecutor, dataSource, fhirEncoders, serverConfiguration); resolver = new SqlDependencyResolver( viewResolver, libraryReferenceResolver, new SqlLibraryParser(), serverConfiguration); @@ -94,49 +110,49 @@ void tearDown() { @Test void storedViewDefinitionDependencyRequiresViewDefinitionRead() { - when(readExecutor.read("ViewDefinition", "pv")) - .thenReturn(simpleViewDefinition("pv", "Patient")); + when(dataSource.read("ViewDefinition")) + .thenReturn(viewDefinitionDataset(simpleViewDefinition("pv", PV_URL, "Patient"))); // Projected-resource READ alone is not enough; the ViewDefinition metadata READ is required. setSecurityContext("pathling:read:Patient"); - assertThatThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of())) + assertThatThrownBy(() -> resolver.resolve(sqlQuery(PV_URL), Map.of())) .isInstanceOf(AccessDeniedError.class) .hasMessageContaining("ViewDefinition"); setSecurityContext("pathling:read:ViewDefinition", "pathling:read:Patient"); - assertThatNoException() - .isThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of())); + assertThatNoException().isThrownBy(() -> resolver.resolve(sqlQuery(PV_URL), Map.of())); } @Test void projectedResourceReadStillRequiredAtTheLeaf() { - when(readExecutor.read("ViewDefinition", "pv")) - .thenReturn(simpleViewDefinition("pv", "Patient")); + when(dataSource.read("ViewDefinition")) + .thenReturn(viewDefinitionDataset(simpleViewDefinition("pv", PV_URL, "Patient"))); // ViewDefinition READ without the projected Patient READ is still denied. setSecurityContext("pathling:read:ViewDefinition"); - assertThatThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of())) + assertThatThrownBy(() -> resolver.resolve(sqlQuery(PV_URL), Map.of())) .isInstanceOf(AccessDeniedError.class) .hasMessageContaining("Patient"); } @Test void storedSqlViewDependencyRequiresLibraryRead() { - final Library base = SqlLibraryFixtures.sqlView("SELECT * FROM pv", "pv", "ViewDefinition/pv"); + final Library base = + SqlLibraryFixtures.sqlViewWithUrl(BASE_URL, "SELECT * FROM pv", "pv", PV_URL); base.setId("base"); - when(readExecutor.read("Library", "base")).thenReturn(base); - when(readExecutor.read("ViewDefinition", "pv")) - .thenReturn(simpleViewDefinition("pv", "Patient")); + when(dataSource.read("Library")).thenReturn(libraryDataset(base)); + when(dataSource.read("ViewDefinition")) + .thenReturn(viewDefinitionDataset(simpleViewDefinition("pv", PV_URL, "Patient"))); // Holding the transitive ViewDefinition and projected reads, but not Library READ, is denied. setSecurityContext("pathling:read:ViewDefinition", "pathling:read:Patient"); - assertThatThrownBy(() -> resolver.resolve(sqlQuery("Library/base"), Map.of())) + assertThatThrownBy(() -> resolver.resolve(sqlQuery(BASE_URL), Map.of())) .isInstanceOf(AccessDeniedError.class) .hasMessageContaining("Library"); setSecurityContext( "pathling:read:Library", "pathling:read:ViewDefinition", "pathling:read:Patient"); - assertThatNoException().isThrownBy(() -> resolver.resolve(sqlQuery("Library/base"), Map.of())); + assertThatNoException().isThrownBy(() -> resolver.resolve(sqlQuery(BASE_URL), Map.of())); } @Test @@ -150,7 +166,7 @@ void inlineSuppliedViewRequiresNoMetadataRead() { .build(); assertThatNoException() - .isThrownBy(() -> resolver.resolve(sqlQuery("ViewDefinition/pv"), Map.of("pv", supplied))); + .isThrownBy(() -> resolver.resolve(sqlQuery(PV_URL), Map.of(PV_URL, supplied))); } @Test @@ -170,6 +186,22 @@ void topLevelQueryReferenceRequiresLibraryRead() { assertThat(libraryReferenceResolver.resolve(new Reference("Library/base"))).isNotNull(); } + // --------------------------------------------------------------------------- + // Helpers. + // --------------------------------------------------------------------------- + + @Nonnull + private Dataset viewDefinitionDataset(@Nonnull final ViewDefinitionResource... views) { + return spark + .createDataset(List.of(views), fhirEncoders.of(ViewDefinitionResource.class)) + .toDF(); + } + + @Nonnull + private Dataset libraryDataset(@Nonnull final Library... libraries) { + return spark.createDataset(List.of(libraries), fhirEncoders.of("Library")).toDF(); + } + @Nonnull private static ParsedSqlQuery sqlQuery(@Nonnull final String resource) { return new ParsedSqlQuery( @@ -188,9 +220,10 @@ private void setSecurityContext(final String... authorities) { @Nonnull private static ViewDefinitionResource simpleViewDefinition( - @Nonnull final String id, @Nonnull final String resourceType) { + @Nonnull final String id, @Nonnull final String url, @Nonnull final String resourceType) { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(id); + view.setUrl(url); view.setName(new StringType(id + "_view")); view.setResource(new CodeType(resourceType)); view.setStatus(new CodeType("active")); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java index 24ca1ae14f..636f69748d 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportFormatIT.java @@ -254,7 +254,7 @@ private Map failingLibrary() { "label", "patients", "resource", - "ViewDefinition/" + SqlQueryExportTestConfiguration.PATIENT_VIEW_ID))); + SqlQueryExportTestConfiguration.PATIENT_VIEW_URL))); return library; } } diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java index ade03ab139..514d795735 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportProviderIT.java @@ -344,16 +344,17 @@ private Map inlineSqlLibrary() { "label", "patients", "resource", - "ViewDefinition/" + SqlQueryExportTestConfiguration.PATIENT_VIEW_ID))); + SqlQueryExportTestConfiguration.PATIENT_VIEW_URL))); return library; } - /** An inline Patient ViewDefinition whose id matches the inline query's relatedArtifact. */ + /** An inline Patient ViewDefinition whose url matches the inline query's relatedArtifact. */ @Nonnull private Map inlineViewDefinition() { final Map view = new LinkedHashMap<>(); view.put("resourceType", "ViewDefinition"); view.put("id", SqlQueryExportTestConfiguration.PATIENT_VIEW_ID); + view.put("url", SqlQueryExportTestConfiguration.PATIENT_VIEW_URL); view.put("name", "supplied_patient_view"); view.put("resource", "Patient"); view.put("status", "active"); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java index e982903977..b20c1c05c2 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryExportTestConfiguration.java @@ -69,9 +69,17 @@ public class SqlQueryExportTestConfiguration { /** Id of the stored Patient ViewDefinition referenced by the patient query. */ public static final String PATIENT_VIEW_ID = "patient-bp"; + /** Canonical URL of the stored Patient ViewDefinition (final segment differs from its id). */ + public static final String PATIENT_VIEW_URL = + "https://pathling.csiro.au/test/ViewDefinition/PatientBp"; + /** Id of the stored Observation ViewDefinition referenced by the observation query. */ public static final String OBSERVATION_VIEW_ID = "observation-weight"; + /** Canonical URL of the stored Observation ViewDefinition. */ + public static final String OBSERVATION_VIEW_URL = + "https://pathling.csiro.au/test/ViewDefinition/ObservationWeight"; + /** Id of the stored SQLQuery Library that selects patients. */ public static final String PATIENT_QUERY_ID = "patient-bp-query"; @@ -103,14 +111,14 @@ public QueryableDataSource deltaLake( "patient_bp_query", "SELECT id, family_name FROM patients ORDER BY id", "patients", - "ViewDefinition/" + PATIENT_VIEW_ID)); + PATIENT_VIEW_URL)); resources.add( sqlLibrary( OBSERVATION_QUERY_ID, "observation_weight_query", "SELECT id, subject, weight_kg FROM observations ORDER BY id", "observations", - "ViewDefinition/" + OBSERVATION_VIEW_ID)); + OBSERVATION_VIEW_URL)); resources.add(parameterisedPatientQuery()); // p1 has an older meta.lastUpdated than p2/p3, so a `_since` filter can scope it out. resources.add(patient("p1", "Smith", P1_LAST_UPDATED)); @@ -128,6 +136,7 @@ public QueryableDataSource deltaLake( private static ViewDefinitionResource patientView() { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(PATIENT_VIEW_ID); + view.setUrl(PATIENT_VIEW_URL); view.setName(new StringType("patient_view")); view.setResource(new CodeType("Patient")); view.setStatus(new CodeType("active")); @@ -142,6 +151,7 @@ private static ViewDefinitionResource patientView() { private static ViewDefinitionResource observationView() { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(OBSERVATION_VIEW_ID); + view.setUrl(OBSERVATION_VIEW_URL); view.setName(new StringType("observation_view")); view.setResource(new CodeType("Observation")); view.setStatus(new CodeType("active")); @@ -162,7 +172,7 @@ private static Library parameterisedPatientQuery() { "patient_by_family_query", "SELECT id, family_name FROM patients WHERE family_name = :familyName", "patients", - "ViewDefinition/" + PATIENT_VIEW_ID); + PATIENT_VIEW_URL); library.addParameter( new ParameterDefinition().setName("familyName").setUse(ParameterUse.IN).setType("string")); return library; diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java index 4ee2c9cc0a..ec6a516b0a 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java @@ -128,10 +128,9 @@ void cleanup() throws IOException { */ @Test void runsSqlAgainstDeltaBackedViewDefinition() { - final String viewId = createPatientViewDefinition(); + final String viewUrl = createPatientViewDefinition(); final Library library = - sqlQueryLibrary( - "SELECT id FROM patients ORDER BY id LIMIT 5", "patients", "ViewDefinition/" + viewId); + sqlQueryLibrary("SELECT id FROM patients ORDER BY id LIMIT 5", "patients", viewUrl); final String body = postOk( @@ -149,10 +148,16 @@ void runsSqlAgainstDeltaBackedViewDefinition() { } } + /** + * Creates a Patient ViewDefinition carrying a canonical url whose final segment differs from the + * server-assigned logical id, then returns that url so the dependency can be referenced by it. + */ @Nonnull private String createPatientViewDefinition() { + final String viewUrl = "https://pathling.csiro.au/test/ViewDefinition/DeltaPatients"; final Map view = new LinkedHashMap<>(); view.put("resourceType", "ViewDefinition"); + view.put("url", viewUrl); view.put("name", "patients"); view.put("resource", "Patient"); view.put("status", "active"); @@ -163,21 +168,15 @@ private String createPatientViewDefinition() { select.put("column", List.of(column)); view.put("select", List.of(select)); - final EntityExchangeResult result = - webTestClient - .post() - .uri("http://localhost:" + port + "/fhir/ViewDefinition") - .header("Content-Type", "application/fhir+json") - .bodyValue(GSON.toJson(view)) - .exchange() - .expectStatus() - .isCreated() - .expectBody() - .returnResult(); - final String location = - Objects.requireNonNull(result.getResponseHeaders().getFirst("Location")); - final String[] parts = location.split("/"); - return parts[parts.length - 1]; + webTestClient + .post() + .uri("http://localhost:" + port + "/fhir/ViewDefinition") + .header("Content-Type", "application/fhir+json") + .bodyValue(GSON.toJson(view)) + .exchange() + .expectStatus() + .isCreated(); + return viewUrl; } @Nonnull diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java index c4a92dfa21..4beb512d65 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunWithViewDefinitionsIT.java @@ -79,10 +79,10 @@ class SqlQueryRunWithViewDefinitionsIT { private static final Gson GSON = new Gson(); private static final String VIEW_REFERENCE = - "ViewDefinition/" + SqlQueryViewDefinitionTestConfiguration.PATIENT_VIEW_ID; + SqlQueryViewDefinitionTestConfiguration.PATIENT_VIEW_URL; private static final String OBSERVATION_VIEW_REFERENCE = - "ViewDefinition/" + SqlQueryViewDefinitionTestConfiguration.OBSERVATION_VIEW_ID; + SqlQueryViewDefinitionTestConfiguration.OBSERVATION_VIEW_URL; @LocalServerPort int port; @@ -186,9 +186,12 @@ void runsSqlAgainstViewWithPathlingUdfInFhirPath() { } @Test - void returns400WhenReferencedViewDefinitionDoesNotExist() { + void returnsErrorWhenReferencedViewDefinitionDoesNotExist() { final Library library = - sqlQueryLibrary("SELECT id FROM patients", "patients", "ViewDefinition/does-not-exist"); + sqlQueryLibrary( + "SELECT id FROM patients", + "patients", + "https://pathling.csiro.au/test/ViewDefinition/does-not-exist"); webTestClient .post() diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryViewDefinitionTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryViewDefinitionTestConfiguration.java index 1a774a486f..56366c3074 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryViewDefinitionTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryViewDefinitionTestConfiguration.java @@ -65,6 +65,14 @@ public class SqlQueryViewDefinitionTestConfiguration { /** The id of the pre-loaded Patient ViewDefinition referenced by tests. */ public static final String PATIENT_VIEW_ID = "patient-view"; + /** + * The canonical URL of the pre-loaded Patient ViewDefinition. The URL's final segment ({@code + * Patients}) deliberately differs from the logical id ({@link #PATIENT_VIEW_ID}), so resolving a + * dependency by this URL exercises the case the id-based resolution could not handle. + */ + public static final String PATIENT_VIEW_URL = + "https://pathling.csiro.au/test/ViewDefinition/Patients"; + /** * The id of the pre-loaded Observation ViewDefinition referenced by tests. Its FHIRPath uses * {@code .toString()} on a Decimal, which compiles to the Pathling-registered {@code @@ -72,6 +80,10 @@ public class SqlQueryViewDefinitionTestConfiguration { */ public static final String OBSERVATION_VIEW_ID = "observation-view"; + /** The canonical URL of the pre-loaded Observation ViewDefinition. */ + public static final String OBSERVATION_VIEW_URL = + "https://pathling.csiro.au/test/ViewDefinition/Observations"; + @Primary @Bean @Nonnull @@ -95,6 +107,7 @@ public QueryableDataSource deltaLake( private static ViewDefinitionResource patientView() { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(PATIENT_VIEW_ID); + view.setUrl(PATIENT_VIEW_URL); view.setName(new StringType("patient_view")); view.setResource(new CodeType("Patient")); view.setStatus(new CodeType("active")); @@ -115,6 +128,7 @@ private static ViewDefinitionResource patientView() { private static ViewDefinitionResource observationView() { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(OBSERVATION_VIEW_ID); + view.setUrl(OBSERVATION_VIEW_URL); view.setName(new StringType("observation_view")); view.setResource(new CodeType("Observation")); view.setStatus(new CodeType("active")); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java index dc7f11283e..2c2d3c48ec 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewRunProviderIT.java @@ -112,7 +112,7 @@ void runsSqlQueryComposingAStoredSqlView() { sqlQueryLibrary( "SELECT id, family_name FROM ap ORDER BY id", "ap", - "Library/" + SqlViewTestConfiguration.ACTIVE_PATIENTS_ID); + SqlViewTestConfiguration.libraryUrl(SqlViewTestConfiguration.ACTIVE_PATIENTS_ID)); final String body = postOk(parametersJson(library)); @@ -135,7 +135,7 @@ void runsSqlQueryComposingANestedSqlViewChain() { sqlQueryLibrary( "SELECT id, family_name FROM rp ORDER BY id", "rp", - "Library/" + SqlViewTestConfiguration.REFINED_PATIENTS_ID); + SqlViewTestConfiguration.libraryUrl(SqlViewTestConfiguration.REFINED_PATIENTS_ID)); final String body = postOk(parametersJson(library)); @@ -163,12 +163,14 @@ void runsSqlQueryOverADiamondOfSqlViews() { new RelatedArtifact() .setType(RelatedArtifactType.DEPENDSON) .setLabel("l") - .setResource("Library/" + SqlViewTestConfiguration.LEFT_PATIENTS_ID)); + .setResource( + SqlViewTestConfiguration.libraryUrl(SqlViewTestConfiguration.LEFT_PATIENTS_ID))); library.addRelatedArtifact( new RelatedArtifact() .setType(RelatedArtifactType.DEPENDSON) .setLabel("r") - .setResource("Library/" + SqlViewTestConfiguration.RIGHT_PATIENTS_ID)); + .setResource( + SqlViewTestConfiguration.libraryUrl(SqlViewTestConfiguration.RIGHT_PATIENTS_ID))); final String body = postOk(parametersJson(library)); @@ -206,9 +208,12 @@ void runsSqlViewByQueryReferenceAtSystemLevel() { } @Test - void returns400WhenReferencedSqlViewDoesNotExist() { + void returnsErrorWhenReferencedSqlViewDoesNotExist() { final Library library = - sqlQueryLibrary("SELECT id FROM missing", "missing", "Library/does-not-exist"); + sqlQueryLibrary( + "SELECT id FROM missing", + "missing", + SqlViewTestConfiguration.libraryUrl("does-not-exist")); // The error names the failing label and reference so the client can act on it. final String body = postExpect4xx(parametersJson(library)); @@ -219,7 +224,10 @@ void returns400WhenReferencedSqlViewDoesNotExist() { void rejectsCyclicSqlViewGraphWith400() { // cycle-a -> cycle-b -> cycle-a must be rejected before any SQL executes. final Library library = - sqlQueryLibrary("SELECT * FROM a", "a", "Library/" + SqlViewTestConfiguration.CYCLE_A_ID); + sqlQueryLibrary( + "SELECT * FROM a", + "a", + SqlViewTestConfiguration.libraryUrl(SqlViewTestConfiguration.CYCLE_A_ID)); final String body = postExpect4xx(parametersJson(library)); assertThat(body).containsIgnoringCase("cycl"); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java index 3f98baa0b9..a655f1c453 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlViewTestConfiguration.java @@ -69,9 +69,34 @@ @TestConfiguration public class SqlViewTestConfiguration { + /** Base canonical URL for test ViewDefinitions. */ + private static final String VIEW_DEFINITION_BASE = + "https://pathling.csiro.au/test/ViewDefinition/"; + + /** Base canonical URL for test SQLView Libraries. */ + private static final String LIBRARY_BASE = "https://pathling.csiro.au/test/Library/"; + /** The id of the pre-loaded Patient ViewDefinition. */ public static final String PATIENT_VIEW_ID = "patient-view"; + /** + * The canonical URL of the pre-loaded Patient ViewDefinition. The URL's final segment ({@code + * Patients}) differs from the logical id, so dependencies that resolve it by URL exercise the + * case the id-based resolution could not handle. + */ + public static final String PATIENT_VIEW_URL = VIEW_DEFINITION_BASE + "Patients"; + + /** + * Returns the canonical URL of a stored SQLView Library given its local id. + * + * @param id the SQLView's logical id + * @return the SQLView's canonical url + */ + @Nonnull + public static String libraryUrl(@Nonnull final String id) { + return LIBRARY_BASE + id; + } + /** The id of the SQLView over the Patient ViewDefinition. */ public static final String ACTIVE_PATIENTS_ID = "active-patients"; @@ -106,27 +131,27 @@ public QueryableDataSource deltaLake( sqlView( ACTIVE_PATIENTS_ID, "SELECT id, family_name FROM patient_view", - Map.of("patient_view", "ViewDefinition/" + PATIENT_VIEW_ID))); + Map.of("patient_view", PATIENT_VIEW_URL))); resources.add( sqlView( REFINED_PATIENTS_ID, "SELECT id, family_name FROM ap WHERE family_name <> 'Johnson'", - Map.of("ap", "Library/" + ACTIVE_PATIENTS_ID))); - resources.add(sqlView(CYCLE_A_ID, "SELECT * FROM b", Map.of("b", "Library/" + CYCLE_B_ID))); - resources.add(sqlView(CYCLE_B_ID, "SELECT * FROM a", Map.of("a", "Library/" + CYCLE_A_ID))); + Map.of("ap", libraryUrl(ACTIVE_PATIENTS_ID)))); + resources.add(sqlView(CYCLE_A_ID, "SELECT * FROM b", Map.of("b", libraryUrl(CYCLE_B_ID)))); + resources.add(sqlView(CYCLE_B_ID, "SELECT * FROM a", Map.of("a", libraryUrl(CYCLE_A_ID)))); resources.add( sqlView( SHARED_PATIENTS_ID, "SELECT id, family_name FROM patient_view", - Map.of("patient_view", "ViewDefinition/" + PATIENT_VIEW_ID))); + Map.of("patient_view", PATIENT_VIEW_URL))); resources.add( sqlView( LEFT_PATIENTS_ID, "SELECT id, family_name FROM sp", - Map.of("sp", "Library/" + SHARED_PATIENTS_ID))); + Map.of("sp", libraryUrl(SHARED_PATIENTS_ID)))); resources.add( sqlView( - RIGHT_PATIENTS_ID, "SELECT id FROM sp", Map.of("sp", "Library/" + SHARED_PATIENTS_ID))); + RIGHT_PATIENTS_ID, "SELECT id FROM sp", Map.of("sp", libraryUrl(SHARED_PATIENTS_ID)))); resources.add(patient("p1", "Smith")); resources.add(patient("p2", "Johnson")); resources.add(patient("p3", "Williams")); @@ -137,6 +162,7 @@ public QueryableDataSource deltaLake( private static ViewDefinitionResource patientView() { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(PATIENT_VIEW_ID); + view.setUrl(PATIENT_VIEW_URL); view.setName(new StringType("patient_view")); view.setResource(new CodeType("Patient")); view.setStatus(new CodeType("active")); @@ -158,7 +184,7 @@ private static Library sqlView( @Nonnull final Map dependenciesByLabel) { final Library library = new Library(); library.setId(id); - library.setUrl("https://pathling.csiro.au/test/Library/" + id); + library.setUrl(libraryUrl(id)); library.setStatus(PublicationStatus.ACTIVE); library.setType( new CodeableConcept() diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java index bd5356effd..13c1212e6e 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java @@ -104,6 +104,31 @@ void resolveTempViewNameDerivesFromCanonicalKeyNotLabel() { assertThat(name).startsWith("sqlquery_req1_").doesNotContain("/").doesNotContain("-"); } + @Test + void resolveTempViewNameDerivesFromCanonicalUrlKey() { + // The canonical key is now a full canonical URL (optionally url|version); the scheme, slashes, + // dots, and version pipe must all sanitise into a legal Spark identifier. + final String name = + ViewRegistrationService.resolveTempViewName( + "req1", "https://example.org/ViewDefinition/Patients|2"); + assertThat(name) + .startsWith("sqlquery_req1_") + .doesNotContain("/") + .doesNotContain(":") + .doesNotContain(".") + .doesNotContain("|"); + } + + @Test + void resolveTempViewNameGivesDistinctNamesToDistinctCanonicalUrlKeys() { + // A bare-url key and a url|version key must not collapse to the same temp view name. + final String bare = + ViewRegistrationService.resolveTempViewName("req1", "https://example.org/V"); + final String versioned = + ViewRegistrationService.resolveTempViewName("req1", "https://example.org/V|2"); + assertThat(bare).isNotEqualTo(versioned); + } + @Test void resolveTempViewNameGivesDistinctNamesToDistinctKeys() { // Two nodes that happen to share a label but resolve to different resources are keyed by their diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java index 238671bb48..492ab17c68 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewResolverTest.java @@ -18,125 +18,145 @@ package au.csiro.pathling.operations.sqlquery; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import au.csiro.pathling.config.AuthorizationConfiguration; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.encoders.ViewDefinitionResource; import au.csiro.pathling.encoders.ViewDefinitionResource.ColumnComponent; import au.csiro.pathling.encoders.ViewDefinitionResource.SelectComponent; -import au.csiro.pathling.errors.ResourceNotFoundError; -import au.csiro.pathling.read.ReadExecutor; -import au.csiro.pathling.views.FhirView; +import au.csiro.pathling.io.source.DataSource; +import au.csiro.pathling.test.SpringBootUnitTest; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import jakarta.annotation.Nonnull; -import java.util.Map; +import jakarta.annotation.Nullable; +import java.util.List; import java.util.Optional; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; /** - * Unit tests for {@link ViewResolver} covering id extraction, the canonical key it produces, and - * the resolve-versus-try semantics for stored and missing ViewDefinitions. + * Unit tests for {@link ViewResolver}, verifying that a {@code ViewDefinition} dependency resolves + * by matching its canonical {@code url} (and {@code url|version}) - never its logical id - and that + * the resolved canonical key is the resource's url plus its version. + * + * @author John Grimes */ +@SpringBootUnitTest class ViewResolverTest { - private ReadExecutor readExecutor; + private static final String PATIENTS_URL = "https://example.org/Patients"; + + @Autowired private SparkSession spark; + @Autowired private FhirEncoders fhirEncoders; + @Autowired private FhirContext fhirContext; + + private DataSource dataSource; private ViewResolver resolver; @BeforeEach void setUp() { - readExecutor = mock(ReadExecutor.class); + dataSource = mock(DataSource.class); final ServerConfiguration serverConfiguration = new ServerConfiguration(); final AuthorizationConfiguration auth = new AuthorizationConfiguration(); auth.setEnabled(false); serverConfiguration.setAuth(auth); - resolver = new ViewResolver(readExecutor, serverConfiguration, FhirContext.forR4()); + resolver = new ViewResolver(dataSource, fhirEncoders, serverConfiguration, fhirContext); } @Test - void resolvesReferenceByBareId() { - when(readExecutor.read("ViewDefinition", "patient-view")) - .thenReturn(simpleViewDefinition("patient-view", "Patient")); + void resolvesAViewDefinitionByUrlNotLogicalId() { + // The logical id ("vd-abc") deliberately differs from the URL's final segment ("Patients"). + when(dataSource.read("ViewDefinition")) + .thenReturn(dataset(viewDefinition("vd-abc", PATIENTS_URL, null, "active", "Patient"))); - final ResolvedViewDefinition resolved = - resolver.resolveViewDefinition( - new ViewArtifactReference("patients", "patient-view"), Map.of()); + final Optional resolved = + resolver.resolveStoredViewDefinition(new ViewArtifactReference("patients", PATIENTS_URL)); - assertThat(resolved.getCanonicalKey()).isEqualTo("ViewDefinition/patient-view"); - assertThat(resolved.getView().getResource()).isEqualTo("Patient"); + assertThat(resolved).isPresent(); + assertThat(resolved.get().getCanonicalKey()).isEqualTo(PATIENTS_URL); + assertThat(resolved.get().getView().getResource()).isEqualTo("Patient"); } @Test - void extractsIdFromCanonicalUrlForTheKey() { - when(readExecutor.read("ViewDefinition", "obs-view")) - .thenReturn(simpleViewDefinition("obs-view", "Observation")); + void returnsEmptyWhenNoStoredViewDefinitionMatchesTheUrl() { + when(dataSource.read("ViewDefinition")) + .thenReturn(dataset(viewDefinition("vd-abc", PATIENTS_URL, null, "active", "Patient"))); - final ResolvedViewDefinition resolved = - resolver.resolveViewDefinition( - new ViewArtifactReference("obs", "https://example.org/ViewDefinition/obs-view"), - Map.of()); + final Optional resolved = + resolver.resolveStoredViewDefinition( + new ViewArtifactReference("missing", "https://example.org/Missing")); - assertThat(resolved.getCanonicalKey()).isEqualTo("ViewDefinition/obs-view"); - assertThat(resolved.getView().getResource()).isEqualTo("Observation"); + assertThat(resolved).isEmpty(); } @Test - void resolveThrowsWhenViewDefinitionNotFound() { - when(readExecutor.read("ViewDefinition", "missing")) - .thenThrow(new ResourceNotFoundError("not there")); - - assertThatThrownBy( - () -> - resolver.resolveViewDefinition( - new ViewArtifactReference("patients", "missing"), Map.of())) - .isInstanceOf(InvalidRequestException.class) - .hasMessageContaining("patients") - .hasMessageContaining("missing"); + void selectsTheExactVersionWhenOneIsRequested() { + when(dataSource.read("ViewDefinition")) + .thenReturn( + dataset( + viewDefinition("v1", PATIENTS_URL, "1", "active", "Patient"), + viewDefinition("v2", PATIENTS_URL, "2", "active", "Patient"))); + + final Optional resolved = + resolver.resolveStoredViewDefinition( + new ViewArtifactReference("patients", PATIENTS_URL + "|2")); + + assertThat(resolved).isPresent(); + assertThat(resolved.get().getCanonicalKey()).isEqualTo(PATIENTS_URL + "|2"); } @Test - void tryResolveReturnsEmptyWhenViewDefinitionNotFound() { - // The bare-canonical disambiguation relies on an empty result here to fall back to a SQLView. - when(readExecutor.read("ViewDefinition", "active-patients")) - .thenThrow(new ResourceNotFoundError("not there")); + void selectsTheLatestActiveVersionForABareUrl() { + when(dataSource.read("ViewDefinition")) + .thenReturn( + dataset( + viewDefinition("v1", PATIENTS_URL, "1", "retired", "Patient"), + viewDefinition("v2", PATIENTS_URL, "2", "active", "Patient"))); final Optional resolved = - resolver.tryResolveViewDefinition( - new ViewArtifactReference("ap", "active-patients"), Map.of()); + resolver.resolveStoredViewDefinition(new ViewArtifactReference("patients", PATIENTS_URL)); - assertThat(resolved).isEmpty(); + // The resolved canonical key is the chosen resource's url plus its version. + assertThat(resolved).isPresent(); + assertThat(resolved.get().getCanonicalKey()).isEqualTo(PATIENTS_URL + "|2"); } - @Test - void prefersSuppliedViewOverStorage() { - final FhirView supplied = - FhirView.ofResource("Patient") - .select(FhirView.columns(FhirView.column("id", "id"))) - .build(); - - final ResolvedViewDefinition resolved = - resolver.resolveViewDefinition( - new ViewArtifactReference("patients", "ViewDefinition/patient-bp"), - Map.of("patient-bp", supplied)); - - assertThat(resolved.getView()).isSameAs(supplied); - assertThat(resolved.getCanonicalKey()).isEqualTo("ViewDefinition/patient-bp"); + // --------------------------------------------------------------------------- + // Helpers. + // --------------------------------------------------------------------------- + + @Nonnull + private Dataset dataset(@Nonnull final ViewDefinitionResource... views) { + return spark + .createDataset(List.of(views), fhirEncoders.of(ViewDefinitionResource.class)) + .toDF(); } @Nonnull - private static ViewDefinitionResource simpleViewDefinition( - @Nonnull final String id, @Nonnull final String resourceType) { + private static ViewDefinitionResource viewDefinition( + @Nonnull final String id, + @Nonnull final String url, + @Nullable final String version, + @Nonnull final String status, + @Nonnull final String resourceType) { final ViewDefinitionResource view = new ViewDefinitionResource(); view.setId(id); + view.setUrl(url); + if (version != null) { + view.setVersion(version); + } view.setName(new StringType(id + "_view")); view.setResource(new CodeType(resourceType)); - view.setStatus(new CodeType("active")); + view.setStatus(new CodeType(status)); final SelectComponent select = new SelectComponent(); final ColumnComponent column = new ColumnComponent(); column.setName(new StringType("id")); From 462ff56f7ffaf24da164d2c81a59a991d098e308 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 22:55:30 +1000 Subject: [PATCH 042/162] feat: Author SQL on FHIR view references by canonical URL in the admin UI The inline SQL authoring form now binds each table source to its canonical URL and emits that URL as relatedArtifact.resource on save, so queries authored in the UI resolve without manual editing. Sources without a URL are listed but disabled with an explanation, and a stored reference that matches no known source is surfaced verbatim with a not-found note. --- ui/e2e/fixtures/fhirData.ts | 6 +- ui/e2e/sqlQuery.spec.ts | 54 +++++++++++ ui/src/components/sqlOnFhir/SqlQueryForm.tsx | 2 + .../sqlOnFhir/SqlQueryInlineTab.tsx | 90 +++++++++++-------- .../__tests__/SqlQueryInlineTab.test.tsx | 71 +++++++++++---- .../__tests__/sqlQueryFormHelpers.test.ts | 83 +++++------------ .../sqlOnFhir/sqlQueryFormHelpers.ts | 60 +------------ .../hooks/__tests__/sqlQueryHelpers.test.ts | 57 ++++++++++++ ui/src/hooks/sqlQueryHelpers.ts | 57 ++++++++++++ ui/src/hooks/useViewDefinitions.ts | 9 +- ui/src/types/sqlQuery.ts | 29 ++++-- 11 files changed, 334 insertions(+), 184 deletions(-) diff --git a/ui/e2e/fixtures/fhirData.ts b/ui/e2e/fixtures/fhirData.ts index 834c37b2ae..710f8f98d8 100644 --- a/ui/e2e/fixtures/fhirData.ts +++ b/ui/e2e/fixtures/fhirData.ts @@ -269,6 +269,7 @@ export const mockExportManifest: Parameters = { export const mockViewDefinition1 = { resourceType: "ViewDefinition", id: "patient-demographics", + url: "https://pathling.example/ViewDefinition/PatientDemographics", name: "Patient Demographics", resource: "Patient", status: "active", @@ -288,6 +289,7 @@ export const mockViewDefinition1 = { export const mockViewDefinition2 = { resourceType: "ViewDefinition", id: "observation-vitals", + url: "https://pathling.example/ViewDefinition/ObservationVitals", name: "Observation Vitals", resource: "Observation", status: "active", @@ -381,7 +383,7 @@ export const mockSqlQueryLibrary1 = { { type: "depends-on", label: "patients", - resource: "ViewDefinition/patient-demographics", + resource: "https://pathling.example/ViewDefinition/PatientDemographics", }, ], parameter: [{ name: "patient_id", use: "in", type: "string" }], @@ -448,7 +450,7 @@ export const mockSqlViewLibrary1 = { { type: "depends-on", label: "patients", - resource: "ViewDefinition/patient-demographics", + resource: "https://pathling.example/ViewDefinition/PatientDemographics", }, ], }; diff --git a/ui/e2e/sqlQuery.spec.ts b/ui/e2e/sqlQuery.spec.ts index 3f3bd20bfe..e571c14c8d 100644 --- a/ui/e2e/sqlQuery.spec.ts +++ b/ui/e2e/sqlQuery.spec.ts @@ -319,6 +319,60 @@ test.describe("SQL on FHIR page - SQL query mode", () => { ).toHaveAttribute("aria-selected", "true"); }); + test("saves the chosen source by its canonical URL", async ({ page }) => { + await mockMetadata(page); + await mockViewDefinitions(page); + + // Capture the body POSTed to /Library so the persisted reference can be + // asserted to be the source's canonical URL. + let postedBody: string | null = null; + await page.route(/\/Library$/, async (route) => { + if (route.request().method() === "POST") { + postedBody = route.request().postData(); + await route.fulfill({ + status: 201, + contentType: "application/fhir+json", + body: JSON.stringify({ + ...mockSqlQueryLibrary1, + id: "saved-library", + }), + }); + return; + } + await route.fulfill({ + status: 200, + contentType: "application/fhir+json", + body: JSON.stringify(mockSqlQueryLibraryBundle), + }); + }); + + await page.goto("/admin/sql-on-fhir"); + await selectSqlQueryMode(page); + + await page.getByRole("tab", { name: /provide sql/i }).click(); + await page.getByRole("textbox", { name: /library title/i }).fill("By URL"); + await page.getByRole("textbox", { name: /^sql$/i }).fill("SELECT 1"); + await page.getByRole("button", { name: /add view/i }).click(); + await page + .getByRole("textbox", { name: /label for view 1/i }) + .fill("patients"); + await page.getByRole("combobox", { name: /source for view 1/i }).click(); + await page.getByRole("option", { name: "Patient Demographics" }).click(); + + await page.getByRole("button", { name: /save to server/i }).click(); + + await expect( + page.getByRole("tab", { name: /select query/i }), + ).toHaveAttribute("aria-selected", "true"); + expect(postedBody).not.toBeNull(); + const saved = JSON.parse(postedBody as unknown as string) as { + relatedArtifact?: Array<{ resource?: string }>; + }; + expect(saved.relatedArtifact?.[0]?.resource).toBe( + "https://pathling.example/ViewDefinition/PatientDemographics", + ); + }); + test("renders a callout when the server returns 400", async ({ page }) => { await mockMetadata(page); await mockSqlQueryLibraries(page); diff --git a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx index 82bd1e19f7..fc8fe18b81 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx @@ -234,10 +234,12 @@ export function SqlQueryForm({ viewDefinitions={(viewDefinitions ?? []).map((vd) => ({ id: vd.id, name: vd.name, + url: vd.url, }))} sqlViews={(storedViews ?? []).map((view) => ({ id: view.id, name: view.title, + url: view.url, }))} disabled={disabled || isExecuting} /> diff --git a/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx b/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx index c64faf27e6..d0d57d2beb 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx @@ -25,11 +25,12 @@ import { PlusIcon, TrashIcon } from "@radix-ui/react-icons"; import { Box, Button, Flex, IconButton, Select, Text, TextArea, TextField } from "@radix-ui/themes"; +import { findSourceByUrl } from "../../hooks/sqlQueryHelpers"; import { FieldGuidance } from "../FieldGuidance"; import { FieldLabel } from "../FieldLabel"; -import { decodeViewReferenceValue, encodeViewReferenceValue } from "./sqlQueryFormHelpers"; import type { + SourceOption, SqlQueryParameterDeclaration, SqlQueryParameterType, SqlQueryRelatedArtifact, @@ -45,9 +46,45 @@ const PARAMETER_TYPES: SqlQueryParameterType[] = [ "dateTime", ]; -interface SourceOption { - id: string; - name: string; +/** + * Renders a grouped list of selectable table sources for the source picker. + * A source is bound by its canonical URL; one without a URL is rendered + * disabled with an inline explanation, since it cannot satisfy a canonical + * dependency reference. + * + * @param props - The component props. + * @param props.label - The group heading (e.g. "View definitions"). + * @param props.sources - The sources to list in this group. + * @returns The select group, or null when there are no sources. + */ +function SourceSelectGroup({ + label, + sources, +}: Readonly<{ label: string; sources: SourceOption[] }>) { + if (sources.length === 0) { + return null; + } + return ( + + {label} + {sources.map((source) => + source.url ? ( + + {source.name} + + ) : ( + + + {source.name} + + No canonical URL — add a url to reference this view + + + + ), + )} + + ); } interface SqlQueryInlineTabProps { @@ -113,7 +150,7 @@ export function SqlQueryInlineTab({ { rowId: crypto.randomUUID(), label: "", - referenceId: "", + referenceUrl: "", }, ]); }; @@ -214,13 +251,11 @@ export function SqlQueryInlineTab({ )} - handleUpdateTable(table.rowId, decodeViewReferenceValue(value)) - } + onValueChange={(value) => handleUpdateTable(table.rowId, { referenceUrl: value })} disabled={disabled || !hasSources} > - {viewDefinitions.length > 0 && ( - - View definitions - {viewDefinitions.map((vd) => ( - - {vd.name} - - ))} - - )} - {sqlViews.length > 0 && ( - - SQL views - {sqlViews.map((sv) => ( - - {sv.name} - - ))} - - )} + + + {table.referenceUrl && + !findSourceByUrl([...viewDefinitions, ...sqlViews], table.referenceUrl) && ( + + Source not found:{" "} + {table.referenceUrl} + + )} ; - sqlViews?: Array<{ id: string; name: string }>; + viewDefinitions?: SourceOption[]; + sqlViews?: SourceOption[]; } = {}, ) { const user = userEvent.setup(); @@ -75,7 +85,7 @@ function renderTab( const EMPTY_ROW: SqlQueryRelatedArtifact = { rowId: "r1", label: "patients", - referenceId: "", + referenceUrl: "", }; describe("SqlQueryInlineTab", () => { @@ -108,8 +118,8 @@ describe("SqlQueryInlineTab", () => { expect(screen.getByRole("option", { name: "Active patients" })).toBeInTheDocument(); }); - // Selecting a ViewDefinition stamps the row with the view-definition kind. - it("updates the row with a view-definition reference", async () => { + // Selecting a ViewDefinition stamps the row with the source's canonical URL. + it("updates the row with the chosen ViewDefinition's url", async () => { const { user, onTablesChange } = renderTab({ tables: [EMPTY_ROW] }); await user.click(screen.getByRole("combobox", { name: /source for view 1/i })); @@ -118,14 +128,13 @@ describe("SqlQueryInlineTab", () => { expect(onTablesChange).toHaveBeenCalledWith([ expect.objectContaining({ rowId: "r1", - referenceType: "view-definition", - referenceId: "patient-demographics", + referenceUrl: PATIENT_DEMOGRAPHICS_URL, }), ]); }); - // Selecting a SQLView stamps the row with the sql-view kind. - it("updates the row with a sql-view reference", async () => { + // Selecting a SQLView stamps the row with the SQLView's canonical URL. + it("updates the row with the chosen SQLView's url", async () => { const { user, onTablesChange } = renderTab({ tables: [EMPTY_ROW] }); await user.click(screen.getByRole("combobox", { name: /source for view 1/i })); @@ -134,12 +143,40 @@ describe("SqlQueryInlineTab", () => { expect(onTablesChange).toHaveBeenCalledWith([ expect.objectContaining({ rowId: "r1", - referenceType: "sql-view", - referenceId: "active-patients", + referenceUrl: ACTIVE_PATIENTS_URL, }), ]); }); + // A source with no canonical URL is rendered disabled with an explanation and + // cannot be selected, since it could never satisfy a canonical reference. + it("disables a URL-less source with an explanation and prevents selecting it", async () => { + const { user, onTablesChange } = renderTab({ tables: [EMPTY_ROW] }); + + await user.click(screen.getByRole("combobox", { name: /source for view 1/i })); + + const draftOption = screen.getByRole("option", { name: /Draft lab observations/i }); + expect(draftOption).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByText(/No canonical URL/i)).toBeInTheDocument(); + + await user.click(draftOption); + expect(onTablesChange).not.toHaveBeenCalled(); + }); + + // When editing a stored query, a saved URL that matches no known source is + // surfaced verbatim with a "source not found" note. + it("surfaces an unmatched stored reference verbatim", () => { + const unmatchedRow: SqlQueryRelatedArtifact = { + rowId: "r1", + label: "patients", + referenceUrl: "https://example.org/ViewDefinition/Gone", + }; + renderTab({ tables: [unmatchedRow] }); + + expect(screen.getByText(/source not found/i)).toBeInTheDocument(); + expect(screen.getByText("https://example.org/ViewDefinition/Gone")).toBeInTheDocument(); + }); + // With neither ViewDefinitions nor SQLViews, the selector is disabled and // shows a "nothing to reference" placeholder. it("disables the selector when there is nothing to reference", () => { diff --git a/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts b/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts index 664ba9dd89..ce3c8afe95 100644 --- a/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts +++ b/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts @@ -24,43 +24,9 @@ import { buildParameterTypes, canExecuteInlineForm, canSaveInlineForm, - decodeViewReferenceValue, - encodeViewReferenceValue, isRuntimeValueValid, } from "../sqlQueryFormHelpers"; -describe("encodeViewReferenceValue / decodeViewReferenceValue", () => { - // The codec round-trips a view-definition reference. - it("round-trips a view-definition reference", () => { - const value = encodeViewReferenceValue("view-definition", "vd-1"); - expect(value).toBe("view-definition:vd-1"); - expect(decodeViewReferenceValue(value)).toEqual({ - referenceType: "view-definition", - referenceId: "vd-1", - }); - }); - - // The codec round-trips a sql-view reference. - it("round-trips a sql-view reference", () => { - const value = encodeViewReferenceValue("sql-view", "lib-1"); - expect(value).toBe("sql-view:lib-1"); - expect(decodeViewReferenceValue(value)).toEqual({ - referenceType: "sql-view", - referenceId: "lib-1", - }); - }); - - // Splitting on the first colon keeps ids that themselves contain colons - // intact, so the two id namespaces never collide. - it("preserves ids that contain colons", () => { - const value = encodeViewReferenceValue("sql-view", "urn:uuid:abc:def"); - expect(decodeViewReferenceValue(value)).toEqual({ - referenceType: "sql-view", - referenceId: "urn:uuid:abc:def", - }); - }); -}); - describe("buildInlineSqlQueryLibrary", () => { // The assembled Library carries the SQL on FHIR profile, the // sql-query type code and the SQL both Base64-encoded and as plain @@ -73,8 +39,7 @@ describe("buildInlineSqlQueryLibrary", () => { { rowId: "r1", label: "patients", - referenceType: "view-definition", - referenceId: "vd-patients", + referenceUrl: "https://example.org/ViewDefinition/patients", }, ], parameters: [{ rowId: "p1", name: "patient_id", type: "string" }], @@ -98,7 +63,7 @@ describe("buildInlineSqlQueryLibrary", () => { { type: "depends-on", label: "patients", - resource: "ViewDefinition/vd-patients", + resource: "https://example.org/ViewDefinition/patients", }, ]); expect(library.parameter).toEqual([ @@ -106,23 +71,24 @@ describe("buildInlineSqlQueryLibrary", () => { ]); }); - // A view row backed by a SQLView emits a Library/ reference, while a - // ViewDefinition row emits ViewDefinition/. - it("emits the correct reference prefix per source kind", () => { + // Each row emits its chosen source's canonical URL verbatim as the + // relatedArtifact.resource - regardless of whether the source is a + // ViewDefinition or a SQLView - so the saved query round-trips a source by + // URL. + it("emits each row's canonical url as the relatedArtifact resource", () => { const library = buildInlineSqlQueryLibrary({ sql: "SELECT 1", tables: [ { rowId: "r1", label: "patients", - referenceType: "view-definition", - referenceId: "patient-demographics", + referenceUrl: + "https://pathling.example/ViewDefinition/patient_demographics", }, { rowId: "r2", label: "active", - referenceType: "sql-view", - referenceId: "active-patients", + referenceUrl: "https://pathling.example/Library/ObservationPeriod", }, ], parameters: [], @@ -132,12 +98,13 @@ describe("buildInlineSqlQueryLibrary", () => { { type: "depends-on", label: "patients", - resource: "ViewDefinition/patient-demographics", + resource: + "https://pathling.example/ViewDefinition/patient_demographics", }, { type: "depends-on", label: "active", - resource: "Library/active-patients", + resource: "https://pathling.example/Library/ObservationPeriod", }, ]); }); @@ -181,8 +148,7 @@ describe("canExecuteInlineForm", () => { { rowId: "r1", label: "patients", - referenceType: "view-definition", - referenceId: "vd1", + referenceUrl: "https://example.org/V", }, ], parameters: [], @@ -208,25 +174,19 @@ describe("canExecuteInlineForm", () => { canExecuteInlineForm({ sql: "SELECT 1", tables: [ - { - rowId: "r1", - label: "", - referenceType: "view-definition", - referenceId: "vd1", - }, + { rowId: "r1", label: "", referenceUrl: "https://example.org/V" }, ], parameters: [], }), ).toBe(false); }); - // A view row with no source picked (no referenceType / referenceId) is - // incomplete. + // A view row with no source picked (no referenceUrl) is incomplete. it("returns false when a view row has no source selected", () => { expect( canExecuteInlineForm({ sql: "SELECT 1", - tables: [{ rowId: "r1", label: "patients", referenceId: "" }], + tables: [{ rowId: "r1", label: "patients", referenceUrl: "" }], parameters: [], }), ).toBe(false); @@ -241,8 +201,7 @@ describe("canExecuteInlineForm", () => { { rowId: "r1", label: "patients", - referenceType: "sql-view", - referenceId: "lib1", + referenceUrl: "https://example.org/Library/lib1", }, ], parameters: [], @@ -261,8 +220,7 @@ describe("canSaveInlineForm", () => { { rowId: "r1", label: "patients", - referenceType: "view-definition", - referenceId: "vd1", + referenceUrl: "https://example.org/V", }, ], parameters: [], @@ -279,8 +237,7 @@ describe("canSaveInlineForm", () => { { rowId: "r1", label: "patients", - referenceType: "view-definition", - referenceId: "vd1", + referenceUrl: "https://example.org/V", }, ], parameters: [], diff --git a/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts b/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts index 05ee989725..722f4019be 100644 --- a/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts +++ b/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts @@ -29,7 +29,6 @@ import { import { encodeSql } from "../../utils"; import type { - SqlOnFhirReferenceType, SqlQueryLibrary, SqlQueryParameterDeclaration, SqlQueryParameterType, @@ -43,52 +42,6 @@ import type { const SQL_TEXT_EXTENSION = "https://sql-on-fhir.org/ig/StructureDefinition/sql-text"; -/** - * Encodes a view-row source into a single Radix `Select` item value. - * - * A ViewDefinition and a SQLView Library may share a logical id, so the - * value carries the kind as a prefix to keep the two id namespaces - * collision-safe within one dropdown. - * - * @param type - The kind of source (`view-definition` or `sql-view`). - * @param id - The logical id of the source. - * @returns The composite `${type}:${id}` value. - * - * @example - * encodeViewReferenceValue("sql-view", "active-patients"); - * // "sql-view:active-patients" - */ -export function encodeViewReferenceValue( - type: SqlOnFhirReferenceType, - id: string, -): string { - return `${type}:${id}`; -} - -/** - * Decodes a composite `Select` item value back into its kind and id. - * - * Splits on the first `:` only, so ids that themselves contain colons (for - * example URN-style ids) are preserved intact. - * - * @param value - A value produced by {@link encodeViewReferenceValue}. - * @returns The decoded `referenceType` and `referenceId`. - * - * @example - * decodeViewReferenceValue("view-definition:vd-1"); - * // { referenceType: "view-definition", referenceId: "vd-1" } - */ -export function decodeViewReferenceValue(value: string): { - referenceType: SqlOnFhirReferenceType; - referenceId: string; -} { - const separator = value.indexOf(":"); - return { - referenceType: value.slice(0, separator) as SqlOnFhirReferenceType, - referenceId: value.slice(separator + 1), - }; -} - /** * Inputs to {@link buildInlineSqlQueryLibrary}. */ @@ -157,10 +110,9 @@ export function buildInlineSqlQueryLibrary( library.relatedArtifact = input.tables.map((table) => ({ type: "depends-on" as const, label: table.label, - resource: - table.referenceType === "sql-view" - ? `Library/${table.referenceId}` - : `ViewDefinition/${table.referenceId}`, + // The source is referenced by its canonical URL, matched against the + // referenced resource's `url` on the server. + resource: table.referenceUrl, })); } @@ -192,11 +144,7 @@ export function canExecuteInlineForm(input: BuildInlineLibraryInput): boolean { if (input.tables.length === 0) { return false; } - if ( - input.tables.some( - (t) => t.label.trim() === "" || !t.referenceType || !t.referenceId, - ) - ) { + if (input.tables.some((t) => t.label.trim() === "" || !t.referenceUrl)) { return false; } return true; diff --git a/ui/src/hooks/__tests__/sqlQueryHelpers.test.ts b/ui/src/hooks/__tests__/sqlQueryHelpers.test.ts index 81de202db4..9345aec3e3 100644 --- a/ui/src/hooks/__tests__/sqlQueryHelpers.test.ts +++ b/ui/src/hooks/__tests__/sqlQueryHelpers.test.ts @@ -18,12 +18,15 @@ import { describe, expect, it } from "vitest"; import { + findSourceByUrl, libraryToSummary, mapLibraryBundle, parseTabularBody, readSqlQueryResponse, + storedReferencesToViewRows, } from "../sqlQueryHelpers"; +import type { SourceOption } from "../../types/sqlQuery"; import type { Bundle, Library } from "fhir/r4"; const libraryFixture: Library = { @@ -219,3 +222,57 @@ describe("readSqlQueryResponse", () => { expect(result.blob.size).toBe(3); }); }); + +describe("storedReferencesToViewRows", () => { + // Each stored dependency reference becomes a view row carrying the canonical + // URL verbatim, in reference order, with deterministic row ids. + it("maps stored references to view rows by canonical url", () => { + const rows = storedReferencesToViewRows([ + { label: "patients", reference: "https://example.org/Patients" }, + { label: "active", reference: "https://example.org/Library/Active" }, + ]); + + expect(rows).toEqual([ + { + rowId: "stored-row-0", + label: "patients", + referenceUrl: "https://example.org/Patients", + }, + { + rowId: "stored-row-1", + label: "active", + referenceUrl: "https://example.org/Library/Active", + }, + ]); + }); +}); + +describe("findSourceByUrl", () => { + const sources: SourceOption[] = [ + { id: "vd1", name: "Patients", url: "https://example.org/Patients" }, + { id: "vd2", name: "Draft", url: undefined }, + { id: "lib1", name: "Active", url: "https://example.org/Library/Active" }, + ]; + + // A stored canonical URL repopulates the picker by matching a known source. + it("matches a known source by its canonical url", () => { + expect(findSourceByUrl(sources, "https://example.org/Patients")).toEqual({ + id: "vd1", + name: "Patients", + url: "https://example.org/Patients", + }); + }); + + // An unmatched URL is surfaced verbatim (no source carries it), so the + // picker can show a "source not found" note instead of a selection. + it("returns undefined for a url no source carries", () => { + expect( + findSourceByUrl(sources, "https://example.org/Unknown"), + ).toBeUndefined(); + }); + + // A url-less source can never be matched. + it("never matches a url-less source", () => { + expect(findSourceByUrl(sources, "")).toBeUndefined(); + }); +}); diff --git a/ui/src/hooks/sqlQueryHelpers.ts b/ui/src/hooks/sqlQueryHelpers.ts index 411d83c498..9942079e43 100644 --- a/ui/src/hooks/sqlQueryHelpers.ts +++ b/ui/src/hooks/sqlQueryHelpers.ts @@ -32,11 +32,13 @@ import { } from "../utils"; import type { + SourceOption, SqlQueryBinaryResult, SqlQueryLibrary, SqlQueryLibrarySummary, SqlQueryOutputFormat, SqlQueryParameterType, + SqlQueryRelatedArtifact, SqlQueryResult, SqlQueryTabularResult, } from "../types/sqlQuery"; @@ -138,6 +140,7 @@ export function libraryToSummary( return { id, title, + url: library.url, sql, relatedArtifacts, parameters, @@ -145,6 +148,60 @@ export function libraryToSummary( }; } +/** + * Repopulates inline-form view rows from a stored query's dependency + * references. Each `relatedArtifact` is a canonical URL, carried verbatim as + * the row's `referenceUrl`; the picker later matches it back to a known source + * (see {@link findSourceByUrl}) or surfaces it as an unmatched URL. + * + * @param relatedArtifacts - The stored query's decoded dependency references. + * @returns The view rows, in reference order, keyed by deterministic row ids. + * + * @example + * storedReferencesToViewRows([ + * { label: "patients", reference: "https://example.org/Patients" }, + * ]); + * // [{ rowId: "stored-row-0", label: "patients", + * // referenceUrl: "https://example.org/Patients" }] + */ +export function storedReferencesToViewRows( + relatedArtifacts: Array<{ label: string; reference: string }>, +): SqlQueryRelatedArtifact[] { + return relatedArtifacts.map((artifact, index) => ({ + rowId: `stored-row-${index}`, + label: artifact.label, + referenceUrl: artifact.reference, + })); +} + +/** + * Finds the source whose canonical URL matches the given reference URL. + * + * Used to repopulate the picker when editing a stored query: a matched source + * is shown selected by name, while an unmatched URL (no source carries it) is + * surfaced verbatim with a "source not found" note. + * + * @param sources - The known selectable sources. + * @param url - The canonical URL to match. + * @returns The matching source, or `undefined` when none carries that URL. + * + * @example + * findSourceByUrl( + * [{ id: "vd1", name: "Patients", url: "https://example.org/Patients" }], + * "https://example.org/Patients", + * ); + * // { id: "vd1", name: "Patients", url: "https://example.org/Patients" } + */ +export function findSourceByUrl( + sources: SourceOption[], + url: string, +): SourceOption | undefined { + if (!url) { + return undefined; + } + return sources.find((source) => source.url === url); +} + /** * Reads the body of a `$sqlquery-run` response and assembles it into a * format-aware result. diff --git a/ui/src/hooks/useViewDefinitions.ts b/ui/src/hooks/useViewDefinitions.ts index 126f046852..5c59c510de 100644 --- a/ui/src/hooks/useViewDefinitions.ts +++ b/ui/src/hooks/useViewDefinitions.ts @@ -30,6 +30,8 @@ import type { Bundle } from "fhir/r4"; export interface ViewDefinitionSummary { id: string; name: string; + /** Canonical URL (`ViewDefinition.url`), used to reference this view by URL. */ + url?: string; json: string; } @@ -76,10 +78,15 @@ export const useViewDefinitions: UseViewDefinitionsFn = (options) => { }); return ( bundle.entry?.map((e) => { - const resource = e.resource as { id: string; name?: string }; + const resource = e.resource as { + id: string; + name?: string; + url?: string; + }; return { id: resource.id, name: resource.name || resource.id, + url: resource.url, json: JSON.stringify(resource, null, 2), }; }) ?? [] diff --git a/ui/src/types/sqlQuery.ts b/ui/src/types/sqlQuery.ts index fd7b4ce59a..39bbb1cb34 100644 --- a/ui/src/types/sqlQuery.ts +++ b/ui/src/types/sqlQuery.ts @@ -47,25 +47,34 @@ export type SqlQueryParameterType = | "dateTime"; /** - * The kind of stored source a view row references, deciding the reference - * prefix emitted into `relatedArtifact.resource`. + * A selectable table source in the inline authoring form: a stored + * ViewDefinition or SQLView, listed by name and bound by its canonical URL. */ -export type SqlOnFhirReferenceType = "view-definition" | "sql-view"; +export interface SourceOption { + /** Logical id of the source (used only as a stable list key). */ + id: string; + /** Human-readable name shown in the picker. */ + name: string; + /** + * Canonical URL the dependency reference resolves against. A source with no + * URL cannot be referenced and is shown disabled in the picker. + */ + url?: string; +} /** * A `relatedArtifact` entry on a SQLQuery Library, expressed in form-state - * terms. A row binds a SQL table label to a chosen stored source, which may - * be a ViewDefinition (`ViewDefinition/`) or a SQLView (`Library/`). + * terms. A row binds a SQL table label to a chosen stored source by the + * source's canonical URL, which is emitted verbatim as + * `relatedArtifact.resource` on save. */ export interface SqlQueryRelatedArtifact { /** Stable identifier for use as a React `key`. */ rowId: string; /** Table name referenced by the SQL. */ label: string; - /** The kind of the chosen source; `undefined` until a source is picked. */ - referenceType?: SqlOnFhirReferenceType; - /** Logical id of the chosen ViewDefinition or SQLView; empty until picked. */ - referenceId: string; + /** Canonical URL of the chosen source; empty until a source is picked. */ + referenceUrl: string; } /** @@ -98,6 +107,8 @@ export interface SqlQueryLibrarySummary { id: string; /** Human-readable title (`title` or `name`, falling back to the ID). */ title: string; + /** Canonical URL (`Library.url`), used to reference this source by URL. */ + url?: string; /** Decoded SQL text from `Library.content[0].data`. */ sql: string; /** Related-artifact entries with label and ViewDefinition reference. */ From 58165afabc70637d27624b7497221d46096625e9 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 22:59:28 +1000 Subject: [PATCH 043/162] docs: Describe canonical-URL resolution for SQL on FHIR dependencies Update the SQL query run/export documentation and runnable examples to reference ViewDefinitions and SQLViews by canonical URL rather than logical id, note that a referenceable view must carry a url (and that existing ViewDefinitions must be re-ingested), and describe the strict, ambiguous, and not-found error responses. --- .../sqlquery/ViewArtifactReference.java | 17 ++++-- .../examples/sqlquery-run-examples.md | 48 +++++++++++------ site/docs/server/operations/sql-export.md | 15 +++--- site/docs/server/operations/sql-run.md | 53 ++++++++++++------- 4 files changed, 86 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewArtifactReference.java b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewArtifactReference.java index 4422017244..ea82da417b 100644 --- a/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewArtifactReference.java +++ b/server/src/main/java/au/csiro/pathling/operations/sqlquery/ViewArtifactReference.java @@ -21,16 +21,23 @@ import lombok.Value; /** - * Represents a reference to a ViewDefinition dependency within a SQLQuery Library resource. Each - * reference has a label (the table alias used in the SQL) and a canonical URL pointing to the - * ViewDefinition. + * Represents a dependency reference within a SQLQuery or SQLView Library resource (a {@code + * relatedArtifact} of type {@code depends-on}). Each reference has a label (the table name used in + * the SQL) and the canonical URL of the ViewDefinition or SQLView that backs that table. The + * canonical URL is matched against the referenced resource's {@code url} element; the logical id + * plays no part in resolution. + * + * @author John Grimes */ @Value public class ViewArtifactReference { - /** The table alias used in the SQL query to reference this ViewDefinition. */ + /** The table name used in the SQL query to reference this dependency. */ @Nonnull String label; - /** The canonical URL or relative reference to the ViewDefinition resource. */ + /** + * The absolute canonical URL of the referenced ViewDefinition or SQLView, optionally suffixed + * with {@code |version}. + */ @Nonnull String canonicalUrl; } diff --git a/server/src/main/resources/examples/sqlquery-run-examples.md b/server/src/main/resources/examples/sqlquery-run-examples.md index ed6f51a1d5..494586c223 100644 --- a/server/src/main/resources/examples/sqlquery-run-examples.md +++ b/server/src/main/resources/examples/sqlquery-run-examples.md @@ -63,6 +63,7 @@ curl -s -X POST "http://localhost:8080/fhir/ViewDefinition" \ -H "Content-Type: application/fhir+json" \ -d '{ "resourceType": "ViewDefinition", + "url": "https://example.org/ViewDefinition/patients", "name": "patients", "resource": "Patient", "select": [ @@ -91,6 +92,7 @@ curl -s -X POST "http://localhost:8080/fhir/ViewDefinition" \ -H "Content-Type: application/fhir+json" \ -d '{ "resourceType": "ViewDefinition", + "url": "https://example.org/ViewDefinition/conditions", "name": "conditions", "resource": "Condition", "select": [ @@ -107,9 +109,14 @@ curl -s -X POST "http://localhost:8080/fhir/ViewDefinition" \ }' ``` -Note the IDs returned in the responses. Replace `` and -`` in the examples below with these values. You can also -look them up: +Note the IDs returned in the responses. The instance-level `$viewdefinition-run` +examples below use those logical IDs, so replace `` and +`` with them. The `$sqlquery-run` examples reference the views +by their **canonical URL** instead, so replace `` and +`` with the `url` values set above +(`https://example.org/ViewDefinition/patients` and +`https://example.org/ViewDefinition/conditions`). You can look up the stored +resources, including their `url`, with: ```bash curl -s "http://localhost:8080/fhir/ViewDefinition?_count=10" \ @@ -202,8 +209,18 @@ curl -s -X POST "http://localhost:8080/fhir/\$viewdefinition-run" \ The `$sqlquery-run` operation takes a Library resource conforming to the SQLQuery profile. The Library contains Base64-encoded SQL in its `content`, -and references stored ViewDefinitions via `relatedArtifact`. The `label` on -each artifact becomes the table name used in the SQL. +and references stored ViewDefinitions (or SQLViews) via `relatedArtifact`. The +`label` on each artifact becomes the table name used in the SQL. + +Each `relatedArtifact.resource` is the **canonical URL** of a ViewDefinition or +SQLView (optionally suffixed with `|version`), matched against the referenced +resource's `url` element - not its logical id. The placeholders +`` and `` below stand for those canonical +URLs. A ViewDefinition or SQLView must therefore carry a `url` to be +referenceable; ViewDefinitions ingested before URL retention was added must be +re-ingested so their `url` is stored. A reference that is not an absolute +canonical URL is rejected with a 400, and one that matches no stored resource +with a 404. ### Resolving the Library by reference @@ -242,8 +259,7 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run?_format=csv" \ ``` If no Library matches, the server responds with 404. If neither (or both) of -`queryResource` and `queryReference` are supplied, the server responds with -400. +`queryResource` and `queryReference` are supplied, the server responds with 400. ### JOIN patients with conditions @@ -276,8 +292,8 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run?_format=csv" \ \"data\": \"${SQL_B64}\" }], \"relatedArtifact\": [ - {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"ViewDefinition/\"}, - {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"ViewDefinition/\"} + {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"\"}, + {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"\"} ] } }] @@ -326,7 +342,7 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run?_format=csv" \ \"data\": \"${SQL_B64}\" }], \"relatedArtifact\": [ - {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"ViewDefinition/\"} + {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"\"} ] } }] @@ -374,8 +390,8 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run?_format=csv" \ \"data\": \"${SQL_B64}\" }], \"relatedArtifact\": [ - {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"ViewDefinition/\"}, - {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"ViewDefinition/\"} + {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"\"}, + {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"\"} ] } }] @@ -432,8 +448,8 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run?_format=csv" \ \"data\": \"${SQL_B64}\" }], \"relatedArtifact\": [ - {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"ViewDefinition/\"}, - {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"ViewDefinition/\"} + {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"\"}, + {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"\"} ] } }] @@ -462,7 +478,7 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run" \ \"data\": \"${SQL_B64}\" }], \"relatedArtifact\": [ - {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"ViewDefinition/\"} + {\"type\": \"depends-on\", \"label\": \"patients\", \"resource\": \"\"} ] } }] @@ -490,7 +506,7 @@ curl -s -X POST "http://localhost:8080/fhir/\$sqlquery-run?_format=json" \ \"data\": \"${SQL_B64}\" }], \"relatedArtifact\": [ - {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"ViewDefinition/\"} + {\"type\": \"depends-on\", \"label\": \"conditions\", \"resource\": \"\"} ] } }] diff --git a/site/docs/server/operations/sql-export.md b/site/docs/server/operations/sql-export.md index 8a91294f78..ac5d185950 100644 --- a/site/docs/server/operations/sql-export.md +++ b/site/docs/server/operations/sql-export.md @@ -61,12 +61,15 @@ Each `query` part must supply exactly one of `query.queryReference` or `404 Not Found`. A SQLQuery `Library` declares its table sources as `relatedArtifact` entries, -each labelling a ViewDefinition the SQL references. The optional `view` -parameter supplies those ViewDefinitions at request time, matched to the -`relatedArtifact` entries by ViewDefinition id. A view the SQL references but no -`view` part supplies is read from server storage, exactly as the synchronous -operation does. Each `view` part must supply exactly one of `view.viewReference` -or `view.viewResource`; supplying both, or neither, returns `400 Bad Request`. A +each labelling a ViewDefinition or SQLView the SQL references by its canonical +URL. The optional `view` parameter supplies those ViewDefinitions at request +time, matched to the `relatedArtifact` entries by canonical URL; a supplied view +is therefore preferred over storage when its `url` matches, and a supplied view +that carries no `url` is rejected with `400 Bad Request` because it could never +satisfy a canonical reference. A view the SQL references but no `view` part +supplies is read from server storage, exactly as the synchronous operation does. +Each `view` part must supply exactly one of `view.viewReference` or +`view.viewResource`; supplying both, or neither, returns `400 Bad Request`. A supplied ViewDefinition that is well-formed but semantically invalid returns `422 Unprocessable Entity`. diff --git a/site/docs/server/operations/sql-run.md b/site/docs/server/operations/sql-run.md index 98aac8a0f0..f149cef0a3 100644 --- a/site/docs/server/operations/sql-run.md +++ b/site/docs/server/operations/sql-run.md @@ -73,9 +73,9 @@ profile. The relevant elements are: - `content` - exactly one entry with `contentType` of `application/sql` and the SQL text Base64-encoded in `data`. - `relatedArtifact` - one entry per dependency the query references. The `label` - becomes the table name available to the SQL, and `resource` points to a - ViewDefinition or a SQLView (relative literal or canonical reference). See - [Composing SQLViews](#composing-sqlviews). + becomes the table name available to the SQL, and `resource` is the canonical + URL of a ViewDefinition or a SQLView, matched against that resource's `url`. + See [Composing SQLViews](#composing-sqlviews). - `parameter` - optional declarations of named runtime parameters. Each entry with `use` of `in` must have a `name` and `type`, and the type must be a primitive FHIR type. A SQLView declares no parameters. @@ -104,7 +104,7 @@ Example Library: { "type": "depends-on", "label": "patients", - "resource": "ViewDefinition/patient-demographics" + "resource": "https://example.org/ViewDefinition/patient-demographics" } ] } @@ -152,7 +152,7 @@ Accept: application/x-ndjson { "type": "depends-on", "label": "patients", - "resource": "ViewDefinition/patient-demographics" + "resource": "https://example.org/ViewDefinition/patient-demographics" } ] } @@ -296,20 +296,33 @@ parameter-less query and returns its rows. ### Reference resolution -Each `relatedArtifact.resource` is resolved as follows: - -- `ViewDefinition/[id]` resolves a ViewDefinition by logical id. -- `Library/[id]` resolves a SQLView Library by logical id. -- A bare canonical (`[url]` or `[url]|[version]`) resolves a ViewDefinition - first, by the canonical's final path segment as id; if none exists, it falls - back to a SQLView Library matched by canonical `url`. - -An explicit relative type prefix is authoritative - the server does not fall -back to the other type when a prefixed reference fails to resolve. A reference -that resolves to neither, a `Library` that is a `sql-query` rather than a -`sql-view`, a cycle (for example `A -> B -> A`), and a graph nested deeper than +Each `relatedArtifact.resource` is an absolute **canonical URL** of a +ViewDefinition or SQLView, resolved by matching the referenced resource's `url` +element - never its logical id: + +- The reference must be an absolute canonical URL (scheme `http://`, `https://` + or `urn:`), optionally suffixed with a single `|version`. A relative literal + form such as `ViewDefinition/abc`, a bare id, or a value carrying a fragment + is rejected with a `400` at parse time. +- `[url]|[version]` selects the resource whose `url` and `version` match + exactly; a bare `[url]` selects the latest active match (preferring `active` + status, then the greatest version string). +- ViewDefinitions are searched first, then SQLView Libraries. A canonical that + matches both a ViewDefinition and a SQLView is rejected as ambiguous (`400`); + one that matches neither is a not-found error (`404`). Both messages name the + failing label and reference. + +A ViewDefinition or SQLView must therefore carry a `url` to be referenceable as +a dependency. ViewDefinitions ingested before URL retention was added must be +re-ingested so their `url` is stored. + +A `Library` that is a `sql-query` rather than a `sql-view`, a cycle (for example +`A -> B -> A`), and a graph nested deeper than `pathling.sqlQuery.maxDependencyDepth` are each rejected with a `400` before any -SQL executes, with a message identifying the cause. +SQL executes, with a message identifying the cause. De-duplication, cycle +detection, and depth enforcement are keyed on resolved canonical identity, so a +bare-url and a `url|version` reference to the same stored resource materialise +once. When authorisation is enabled, resolving a ViewDefinition from storage requires READ on `ViewDefinition`, and resolving a SQLView from storage requires READ on @@ -443,8 +456,8 @@ def main(): library = build_sql_query_library( sql, { - "patients": "ViewDefinition/patient-demographics", - "conditions": "ViewDefinition/conditions", + "patients": "https://example.org/ViewDefinition/patient-demographics", + "conditions": "https://example.org/ViewDefinition/conditions", }, ) From 52ad991b0aa25da2d0b2c9f4b85c213ae9488485 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 27 Jun 2026 23:17:13 +1000 Subject: [PATCH 044/162] fix: Use a spaced hyphen in the URL-less source explanation Align the inline picker's disabled-source note with the wireframe's spaced hyphen instead of an em-dash. --- ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx b/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx index d0d57d2beb..fec25c5808 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryInlineTab.tsx @@ -77,7 +77,7 @@ function SourceSelectGroup({ {source.name} - No canonical URL — add a url to reference this view + No canonical URL - add a url to reference this view From 2b0630011c82bef2bc6c4e8ca56b058c900c82b7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 28 Jun 2026 08:48:26 +1000 Subject: [PATCH 045/162] fix: Open sun.util.calendar for local server runs The spring-boot:run goal launched the server without --add-opens=java.base/sun.util.calendar=ALL-UNNAMED, so Spark's date conversion failed with an IllegalAccessException whenever a response contained date values (for example a $sqlquery-run returning date columns). The production entrypoint and the test configuration already supplied this option; only the local run path had drifted. Centralise the server JVM module options into the pathling.runtime.jvmModuleOpts and pathling.test.jvmModuleOpts properties so the run, surefire and failsafe configurations share a single definition, and note the coupling with the production entrypoint that a shell script cannot reference at runtime. --- server/pom.xml | 49 ++++++++--------------- server/src/main/jib/usr/bin/entrypoint.sh | 4 +- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/server/pom.xml b/server/pom.xml index 567cdf3436..3d970df468 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -56,6 +56,17 @@ 2.40.3 1.0.4 1.0.0 + + + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + + ${pathling.runtime.jvmModuleOpts} --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED @@ -585,7 +596,9 @@ spring-boot-maven-plugin ${pathling.springBootVersion} - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED + + ${pathling.runtime.jvmModuleOpts} @@ -601,22 +614,7 @@ false - @{argLine} - --add-opens java.base/java.lang=ALL-UNNAMED - --add-opens java.base/java.lang.invoke=ALL-UNNAMED - --add-opens java.base/java.lang.reflect=ALL-UNNAMED - --add-opens java.base/java.io=ALL-UNNAMED - --add-opens java.base/java.net=ALL-UNNAMED - --add-opens java.base/java.nio=ALL-UNNAMED - --add-opens java.base/java.util=ALL-UNNAMED - --add-opens java.base/java.util.concurrent=ALL-UNNAMED - --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens java.base/sun.nio.ch=ALL-UNNAMED - --add-opens java.base/sun.nio.cs=ALL-UNNAMED - --add-opens java.base/sun.security.action=ALL-UNNAMED - --add-opens java.base/sun.util.calendar=ALL-UNNAMED - --add-exports java.base/sun.nio.ch=ALL-UNNAMED - + @{argLine} ${pathling.test.jvmModuleOpts} @@ -639,22 +637,7 @@ junit.jupiter.execution.parallel.enabled=true - @{argLine} - --add-opens java.base/java.lang=ALL-UNNAMED - --add-opens java.base/java.lang.invoke=ALL-UNNAMED - --add-opens java.base/java.lang.reflect=ALL-UNNAMED - --add-opens java.base/java.io=ALL-UNNAMED - --add-opens java.base/java.net=ALL-UNNAMED - --add-opens java.base/java.nio=ALL-UNNAMED - --add-opens java.base/java.util=ALL-UNNAMED - --add-opens java.base/java.util.concurrent=ALL-UNNAMED - --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens java.base/sun.nio.ch=ALL-UNNAMED - --add-opens java.base/sun.nio.cs=ALL-UNNAMED - --add-opens java.base/sun.security.action=ALL-UNNAMED - --add-opens java.base/sun.util.calendar=ALL-UNNAMED - --add-exports java.base/sun.nio.ch=ALL-UNNAMED - + @{argLine} ${pathling.test.jvmModuleOpts} diff --git a/server/src/main/jib/usr/bin/entrypoint.sh b/server/src/main/jib/usr/bin/entrypoint.sh index ee4ed77032..074376b72e 100755 --- a/server/src/main/jib/usr/bin/entrypoint.sh +++ b/server/src/main/jib/usr/bin/entrypoint.sh @@ -18,7 +18,9 @@ set -e # The Pathling JVM module options required on Java 21. Both roles must run with # these, so they are defined once here and referenced from each branch. They # were previously supplied by the Jib build's jvmFlags, which Jib ignores once -# an explicit entrypoint is configured. +# an explicit entrypoint is configured. This list must be kept in sync with the +# pathling.runtime.jvmModuleOpts property in the server pom.xml, which a shell +# script cannot reference at build or run time. PATHLING_JVM_OPTS=( --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED From 2d807f2e1a2c5729b0cbaa038d516dc7427f877e Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 28 Jun 2026 10:30:23 +1000 Subject: [PATCH 046/162] fix: Show submitted SQL for stored SQL on FHIR queries The result card left the "Submitted SQL" section blank when running a stored SQLQuery or SQLView, because the page only captured the SQL text for inline requests. The form already resolves the selected library's SQL, so carry it on the stored request (for display only, since the server still receives just the reference) and recover it through a shared request-to-SQL helper. The helper also decodes the Base64 content as a fallback for inline requests lacking a sql-text extension. --- ui/src/components/sqlOnFhir/SqlQueryForm.tsx | 3 + .../sqlOnFhir/__tests__/SqlQueryForm.test.tsx | 38 +++++++++ .../__tests__/sqlQueryFormHelpers.test.ts | 81 ++++++++++++++++++- .../sqlOnFhir/sqlQueryFormHelpers.ts | 30 ++++++- ui/src/pages/SqlOnFhir.tsx | 30 +------ ui/src/types/sqlQuery.ts | 6 ++ 6 files changed, 158 insertions(+), 30 deletions(-) diff --git a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx index fc8fe18b81..2e445dab3e 100644 --- a/ui/src/components/sqlOnFhir/SqlQueryForm.tsx +++ b/ui/src/components/sqlOnFhir/SqlQueryForm.tsx @@ -152,6 +152,9 @@ export function SqlQueryForm({ const request: SqlQueryRequest = { mode: "stored", libraryId: selectedLibraryId, + // Carry the resolved SQL for display only; the server receives just + // the reference. + sql: activeStoredLibrary?.sql, ...baseRequestOptions(), }; onExecute(request); diff --git a/ui/src/components/sqlOnFhir/__tests__/SqlQueryForm.test.tsx b/ui/src/components/sqlOnFhir/__tests__/SqlQueryForm.test.tsx index 318a398148..a21185dedf 100644 --- a/ui/src/components/sqlOnFhir/__tests__/SqlQueryForm.test.tsx +++ b/ui/src/components/sqlOnFhir/__tests__/SqlQueryForm.test.tsx @@ -168,3 +168,41 @@ describe("SqlQueryForm runtime parameter visibility", () => { expect(screen.getByText(RUNTIME_SECTION)).toBeInTheDocument(); }); }); + +describe("SqlQueryForm stored execution", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + // Executing a stored query attaches the resolved SQL to the request so the + // result card can show what ran, even though only the reference is sent to + // the server. + it("forwards the selected query's SQL on the request", async () => { + const user = userEvent.setup(); + const onExecute = vi.fn(); + render( + , + ); + + await selectSource(user, "Plain query"); + await user.click(screen.getByRole("button", { name: /execute/i })); + + expect(onExecute).toHaveBeenCalledTimes(1); + expect(onExecute).toHaveBeenCalledWith( + expect.objectContaining({ + mode: "stored", + libraryId: "plain-query", + sql: "SELECT 1", + }), + ); + }); +}); diff --git a/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts b/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts index ce3c8afe95..c28676fbb8 100644 --- a/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts +++ b/ui/src/components/sqlOnFhir/__tests__/sqlQueryFormHelpers.test.ts @@ -17,16 +17,19 @@ import { describe, expect, it } from "vitest"; -import { decodeSql } from "../../../utils/sqlBase64"; +import { decodeSql, encodeSql } from "../../../utils/sqlBase64"; import { areRuntimeBindingsValid, buildInlineSqlQueryLibrary, buildParameterTypes, canExecuteInlineForm, canSaveInlineForm, + extractRequestSql, isRuntimeValueValid, } from "../sqlQueryFormHelpers"; +import type { SqlQueryLibrary, SqlQueryRequest } from "../../../types/sqlQuery"; + describe("buildInlineSqlQueryLibrary", () => { // The assembled Library carries the SQL on FHIR profile, the // sql-query type code and the SQL both Base64-encoded and as plain @@ -331,3 +334,79 @@ describe("buildParameterTypes", () => { ).toEqual({ patient_id: "string", active: "boolean" }); }); }); + +describe("extractRequestSql", () => { + /** + * Builds an inline request wrapping a Library with the supplied content, + * so the recovery paths can be exercised in isolation. + * + * @param content - The `Library.content` array to embed. + * @returns An inline SQL query request. + */ + function inlineRequest(content: SqlQueryLibrary["content"]): SqlQueryRequest { + return { + mode: "inline", + library: { + resourceType: "Library", + status: "active", + type: { + coding: [ + { + system: "https://sql-on-fhir.org/ig/CodeSystem/LibraryTypesCodes", + code: "sql-query", + }, + ], + }, + content, + }, + }; + } + + // A stored request carries the SQL the form resolved from the picker. + it("returns the resolved SQL for a stored request", () => { + expect( + extractRequestSql({ + mode: "stored", + libraryId: "lib-1", + sql: "SELECT 1", + }), + ).toBe("SELECT 1"); + }); + + // A stored request that was built without a resolved SQL yields an empty + // string rather than throwing. + it("returns an empty string for a stored request with no resolved SQL", () => { + expect(extractRequestSql({ mode: "stored", libraryId: "lib-1" })).toBe(""); + }); + + // Inline requests prefer the human-readable sql-text extension. + it("returns the sql-text extension for an inline request", () => { + const request = inlineRequest([ + { + contentType: "application/sql", + data: encodeSql("SELECT 99"), + extension: [ + { + url: "https://sql-on-fhir.org/ig/StructureDefinition/sql-text", + valueString: "SELECT 2", + }, + ], + }, + ]); + expect(extractRequestSql(request)).toBe("SELECT 2"); + }); + + // When no extension is present, inline requests fall back to decoding the + // Base64 data. + it("decodes the Base64 data when no sql-text extension is present", () => { + const request = inlineRequest([ + { contentType: "application/sql", data: encodeSql("SELECT 3") }, + ]); + expect(extractRequestSql(request)).toBe("SELECT 3"); + }); + + // An inline request with no content has no SQL to recover. + it("returns an empty string for an inline request with no content", () => { + expect(extractRequestSql(inlineRequest([]))).toBe(""); + }); +}); diff --git a/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts b/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts index 722f4019be..72d62c7288 100644 --- a/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts +++ b/ui/src/components/sqlOnFhir/sqlQueryFormHelpers.ts @@ -26,13 +26,14 @@ import { SQL_QUERY_LIBRARY_PROFILE, SQL_QUERY_LIBRARY_TYPE_SYSTEM, } from "../../api"; -import { encodeSql } from "../../utils"; +import { decodeSql, encodeSql } from "../../utils"; import type { SqlQueryLibrary, SqlQueryParameterDeclaration, SqlQueryParameterType, SqlQueryRelatedArtifact, + SqlQueryRequest, SqlQueryRuntimeBindings, } from "../../types/sqlQuery"; @@ -127,6 +128,33 @@ export function buildInlineSqlQueryLibrary( return library; } +/** + * Recovers the plain SQL text from a `SqlQueryRequest` for display. + * + * For stored requests the text comes from the resolved `sql` field, which + * the form copies from the selected Library. For inline requests it is read + * from the `sql-text` extension on `Library.content[0]`, falling back to + * decoding the Base64 `data`. Returns the empty string when no SQL can be + * recovered. + * + * @param request - The request whose SQL should be displayed. + * @returns The plain SQL text, or an empty string if it cannot be recovered. + */ +export function extractRequestSql(request: SqlQueryRequest): string { + if (request.mode === "stored") { + return request.sql ?? ""; + } + const content = request.library.content?.[0]; + if (!content) { + return ""; + } + const ext = content.extension?.find((e) => e.url.endsWith("/sql-text")); + if (ext?.valueString) { + return ext.valueString; + } + return content.data ? decodeSql(content.data) : ""; +} + /** * Returns true when the inline form has the minimum data required to * execute a query. diff --git a/ui/src/pages/SqlOnFhir.tsx b/ui/src/pages/SqlOnFhir.tsx index 93755845d9..231a2f36fb 100644 --- a/ui/src/pages/SqlOnFhir.tsx +++ b/ui/src/pages/SqlOnFhir.tsx @@ -29,6 +29,7 @@ import { LoginRequired } from "../components/auth/LoginRequired"; import { SessionExpiredDialog } from "../components/auth/SessionExpiredDialog"; import { SqlOnFhirForm } from "../components/sqlOnFhir/SqlOnFhirForm"; import { SqlQueryCard } from "../components/sqlOnFhir/SqlQueryCard"; +import { extractRequestSql } from "../components/sqlOnFhir/sqlQueryFormHelpers"; import { ViewCard } from "../components/sqlOnFhir/ViewCard"; import { config } from "../config"; import { useAuth } from "../contexts/AuthContext"; @@ -95,12 +96,11 @@ export function SqlOnFhir() { * @param request - The SQL query request. */ const handleExecuteSqlQuery = (request: SqlQueryRequest) => { - const sql = request.mode === "inline" ? extractSqlText(request) : ""; const newJob: SqlQueryJob = { id: crypto.randomUUID(), mode: request.mode, request, - sql, + sql: extractRequestSql(request), createdAt: new Date(), }; setPageJobs((prev) => [{ type: "sql-query", job: newJob }, ...prev]); @@ -180,29 +180,3 @@ export function SqlOnFhir() { ); } - -/** - * Extracts the plain SQL text from an inline `SqlQueryRequest`. - * - * Looks at the `sql-text` extension on `Library.content[0]` first, then - * falls back to decoding `Library.content[0].data`. Returns the empty - * string when neither is available. - * - * @param request - The SQL query request to inspect. - * @returns The plain SQL text, or an empty string if it cannot be - * recovered. - */ -function extractSqlText(request: SqlQueryRequest): string { - if (request.mode !== "inline") { - return ""; - } - const content = request.library.content?.[0]; - if (!content) { - return ""; - } - const ext = content.extension?.find((e) => e.url.endsWith("/sql-text")); - if (ext?.valueString) { - return ext.valueString; - } - return ""; -} diff --git a/ui/src/types/sqlQuery.ts b/ui/src/types/sqlQuery.ts index 39bbb1cb34..3469e9463a 100644 --- a/ui/src/types/sqlQuery.ts +++ b/ui/src/types/sqlQuery.ts @@ -184,6 +184,12 @@ export type SqlQueryRequest = mode: "stored"; /** ID of a stored Library conforming to the SQLQuery profile. */ libraryId: string; + /** + * Resolved SQL text of the referenced Library, retained for display + * only. The server receives just the `libraryId` reference, so this is + * never sent; it lets the result card show the query that ran. + */ + sql?: string; }) | (SqlQueryExecutionOptions & { mode: "inline"; From e26fb951a1781b191bce2a8d24e5e9a7f23ebe8e Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 28 Jun 2026 11:09:04 +1000 Subject: [PATCH 047/162] fix: Bound the height of the submitted SQL in query results The result card rendered the submitted SQL in an unbounded box, so a long query stretched the card far down the page. Extract the existing SQL preview into a shared, height-bounded component with a copy control and reuse it for both the stored-query preview and the submitted SQL echoed in result and error bodies. --- ui/src/components/sqlOnFhir/SqlPreview.tsx | 83 ++++++++++++++++++ ui/src/components/sqlOnFhir/SqlQueryCard.tsx | 27 +----- .../sqlOnFhir/SqlQueryStoredTab.tsx | 45 +--------- .../sqlOnFhir/__tests__/SqlPreview.test.tsx | 84 +++++++++++++++++++ .../sqlOnFhir/__tests__/SqlQueryCard.test.tsx | 25 +++++- 5 files changed, 197 insertions(+), 67 deletions(-) create mode 100644 ui/src/components/sqlOnFhir/SqlPreview.tsx create mode 100644 ui/src/components/sqlOnFhir/__tests__/SqlPreview.test.tsx diff --git a/ui/src/components/sqlOnFhir/SqlPreview.tsx b/ui/src/components/sqlOnFhir/SqlPreview.tsx new file mode 100644 index 0000000000..c6bf00fe21 --- /dev/null +++ b/ui/src/components/sqlOnFhir/SqlPreview.tsx @@ -0,0 +1,83 @@ +/* + * Copyright © 2018-2026 Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shared, height-bounded SQL display. + * + * Renders SQL text in a read-only, scrollable text area with an overlaid copy + * control. Used both for previewing the SQL of a stored query or view in the + * form and for echoing the submitted SQL in a result card, so the two surfaces + * stay visually consistent and neither grows unbounded with long queries. + * + * @author John Grimes + */ + +import { CopyIcon } from "@radix-ui/react-icons"; +import { Box, IconButton, TextArea, Tooltip } from "@radix-ui/themes"; + +import { useClipboard } from "../../hooks"; + +interface SqlPreviewProps { + /** The SQL text to display. */ + sql: string; + /** Accessible label for the read-only text area. */ + ariaLabel: string; + /** Number of visible text rows before the area scrolls. Defaults to 10. */ + rows?: number; +} + +/** + * Renders SQL in a read-only, height-bounded text area with a copy control. + * + * @param props - The component props. + * @param props.sql - The SQL text to display. + * @param props.ariaLabel - Accessible label for the read-only text area. + * @param props.rows - Number of visible text rows before scrolling. Defaults to 10. + * @returns The SQL preview element. + */ +export function SqlPreview({ sql, ariaLabel, rows = 10 }: Readonly) { + const copyToClipboard = useClipboard(); + + return ( + + + copyToClipboard(sql)} + style={{ + position: "absolute", + top: 8, + right: 8, + zIndex: 1, + }} + > + + + +