From c825197dc63dcb35c9db74255733dc03eb380bb3 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Mon, 11 May 2026 20:00:18 +1000 Subject: [PATCH 01/12] chore: Update core version to 9.7.0-SNAPSHOT --- benchmark/pom.xml | 2 +- encoders/pom.xml | 2 +- fhirpath/pom.xml | 2 +- lib/R/pom.xml | 2 +- lib/python/pom.xml | 2 +- library-api/pom.xml | 2 +- library-runtime/pom.xml | 2 +- pom.xml | 2 +- site/pom.xml | 2 +- terminology/pom.xml | 2 +- utilities/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 80a4d8e21e..303af2b009 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -23,7 +23,7 @@ au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT benchmark jar diff --git a/encoders/pom.xml b/encoders/pom.xml index c964319feb..6335501c5c 100644 --- a/encoders/pom.xml +++ b/encoders/pom.xml @@ -32,7 +32,7 @@ au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT encoders jar diff --git a/fhirpath/pom.xml b/fhirpath/pom.xml index 72a21f3610..1d74a8a73d 100644 --- a/fhirpath/pom.xml +++ b/fhirpath/pom.xml @@ -26,7 +26,7 @@ au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT fhirpath jar diff --git a/lib/R/pom.xml b/lib/R/pom.xml index 7ec2c258b3..ff336c0189 100644 --- a/lib/R/pom.xml +++ b/lib/R/pom.xml @@ -26,7 +26,7 @@ au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT ../../pom.xml r diff --git a/lib/python/pom.xml b/lib/python/pom.xml index ebfa4b7bcf..02e5474cfe 100644 --- a/lib/python/pom.xml +++ b/lib/python/pom.xml @@ -26,7 +26,7 @@ au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT ../../pom.xml python diff --git a/library-api/pom.xml b/library-api/pom.xml index 3d9c92b13b..6183af12f5 100644 --- a/library-api/pom.xml +++ b/library-api/pom.xml @@ -26,7 +26,7 @@ pathling au.csiro.pathling - 9.6.0 + 9.7.0-SNAPSHOT library-api jar diff --git a/library-runtime/pom.xml b/library-runtime/pom.xml index d438f9ecf6..e09384dfaf 100644 --- a/library-runtime/pom.xml +++ b/library-runtime/pom.xml @@ -26,7 +26,7 @@ pathling au.csiro.pathling - 9.6.0 + 9.7.0-SNAPSHOT library-runtime jar diff --git a/pom.xml b/pom.xml index 8c0e63e00e..b0ec19d6e5 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ 4.0.0 au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT pom Pathling diff --git a/site/pom.xml b/site/pom.xml index c90e0b254a..6588beafe8 100644 --- a/site/pom.xml +++ b/site/pom.xml @@ -26,7 +26,7 @@ au.csiro.pathling pathling - 9.6.0 + 9.7.0-SNAPSHOT ../pom.xml site diff --git a/terminology/pom.xml b/terminology/pom.xml index b7396389d6..24dd3b624d 100644 --- a/terminology/pom.xml +++ b/terminology/pom.xml @@ -26,7 +26,7 @@ pathling au.csiro.pathling - 9.6.0 + 9.7.0-SNAPSHOT terminology jar diff --git a/utilities/pom.xml b/utilities/pom.xml index b2207fdb39..611f0c05dc 100644 --- a/utilities/pom.xml +++ b/utilities/pom.xml @@ -26,7 +26,7 @@ pathling au.csiro.pathling - 9.6.0 + 9.7.0-SNAPSHOT utilities jar From c0a68d74336dac618d96729961022e473270fe61 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 13 May 2026 14:47:15 +1000 Subject: [PATCH 02/12] fix: Resolve job directory file system from warehouse URI The async job cleanup path used Hadoop's default file system to delete per-job directories, which failed with "Wrong FS" when the warehouse was on a non-default scheme such as s3a://. Route the deletion through the existing JobDirectoryFileSystem so the file system is resolved from the warehouse URI. Fixes #2612. --- .../au/csiro/pathling/async/JobProvider.java | 23 +++++------- .../async/JobProviderSecurityTest.java | 13 ++++--- .../csiro/pathling/async/JobProviderTest.java | 35 +++++++++++++++++-- .../util/FhirServerTestConfiguration.java | 6 ++-- 4 files changed, 52 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/async/JobProvider.java b/server/src/main/java/au/csiro/pathling/async/JobProvider.java index 409f3ff9af..b4d464d7ff 100644 --- a/server/src/main/java/au/csiro/pathling/async/JobProvider.java +++ b/server/src/main/java/au/csiro/pathling/async/JobProvider.java @@ -25,6 +25,7 @@ import au.csiro.pathling.errors.AccessDeniedError; import au.csiro.pathling.errors.ErrorHandlingInterceptor; import au.csiro.pathling.errors.ResourceNotFoundError; +import au.csiro.pathling.io.JobDirectoryFileSystem; import au.csiro.pathling.security.PathlingAuthority; import ca.uhn.fhir.rest.annotation.Operation; import ca.uhn.fhir.rest.annotation.OperationParam; @@ -39,10 +40,8 @@ import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.spark.sql.SparkSession; import org.hl7.fhir.instance.model.api.IBaseOperationOutcome; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.OperationOutcome; @@ -51,7 +50,6 @@ import org.hl7.fhir.r4.model.OperationOutcome.OperationOutcomeIssueComponent; import org.hl7.fhir.r4.model.Parameters; import org.jetbrains.annotations.NotNull; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; @@ -75,27 +73,23 @@ public class JobProvider { @Nonnull private final ServerConfiguration configuration; @Nonnull private final JobRegistry jobRegistry; - private final SparkSession sparkSession; - private final String databasePath; + @Nonnull private final JobDirectoryFileSystem jobDirectoryFileSystem; /** * Creates a new JobProvider. * * @param configuration a {@link ServerConfiguration} for determining if authorisation is enabled * @param jobRegistry the {@link JobRegistry} used to keep track of running jobs - * @param sparkSession the Spark session for file system operations - * @param databasePath the path to the database for job file storage + * @param jobDirectoryFileSystem the {@link JobDirectoryFileSystem} used to resolve and delete + * per-job directories on the warehouse file system */ public JobProvider( @Nonnull final ServerConfiguration configuration, @Nonnull final JobRegistry jobRegistry, - final SparkSession sparkSession, - @Value("${pathling.storage.warehouseUrl}/${pathling.storage.databaseName}") - final String databasePath) { + @Nonnull final JobDirectoryFileSystem jobDirectoryFileSystem) { this.configuration = configuration; this.jobRegistry = jobRegistry; - this.sparkSession = sparkSession; - this.databasePath = new Path(databasePath, "jobs").toString(); + this.jobDirectoryFileSystem = jobDirectoryFileSystem; } /** @@ -213,9 +207,8 @@ private void handleJobDeleteRequest(final Job job) { * @throws IOException if file deletion fails */ public void deleteJobFiles(final String jobId) throws IOException { - final Configuration hadoopConfig = sparkSession.sparkContext().hadoopConfiguration(); - final FileSystem fs = FileSystem.get(hadoopConfig); - final Path jobDirToDel = new Path(databasePath, jobId); + final FileSystem fs = jobDirectoryFileSystem.getFileSystem(); + final Path jobDirToDel = jobDirectoryFileSystem.jobDirectory(jobId); log.debug("Deleting dir {}", jobDirToDel); final boolean deleted = fs.delete(jobDirToDel, true); if (!deleted) { diff --git a/server/src/test/java/au/csiro/pathling/async/JobProviderSecurityTest.java b/server/src/test/java/au/csiro/pathling/async/JobProviderSecurityTest.java index ac45166fd1..1c4853efa3 100644 --- a/server/src/test/java/au/csiro/pathling/async/JobProviderSecurityTest.java +++ b/server/src/test/java/au/csiro/pathling/async/JobProviderSecurityTest.java @@ -25,15 +25,18 @@ import au.csiro.pathling.config.AuthorizationConfiguration; import au.csiro.pathling.config.ServerConfiguration; import au.csiro.pathling.errors.AccessDeniedError; +import au.csiro.pathling.io.JobDirectoryFileSystem; import jakarta.annotation.Nonnull; +import java.nio.file.Path; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import org.apache.spark.sql.SparkSession; +import org.apache.hadoop.conf.Configuration; import org.hl7.fhir.instance.model.api.IBaseResource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.security.core.GrantedAuthority; @@ -57,6 +60,8 @@ class JobProviderSecurityTest { private MockHttpServletRequest request; private MockHttpServletResponse response; + @TempDir Path tempDir; + @BeforeEach void setUp() { jobRegistry = new JobRegistry(); @@ -72,10 +77,10 @@ void setUp() { when(asyncConfig.getCacheMaxAge()).thenReturn(1); when(serverConfiguration.getAsync()).thenReturn(asyncConfig); - // Create mock SparkSession. - final SparkSession sparkSession = mock(SparkSession.class); + final JobDirectoryFileSystem jobDirectoryFileSystem = + new JobDirectoryFileSystem(tempDir.toUri(), new Configuration()); - jobProvider = new JobProvider(serverConfiguration, jobRegistry, sparkSession, "/tmp/test"); + jobProvider = new JobProvider(serverConfiguration, jobRegistry, jobDirectoryFileSystem); request = new MockHttpServletRequest(); response = new MockHttpServletResponse(); diff --git a/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java b/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java index f886fce19c..f8465fe3a9 100644 --- a/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/async/JobProviderTest.java @@ -25,20 +25,24 @@ import au.csiro.pathling.config.AsyncConfiguration; import au.csiro.pathling.config.AuthorizationConfiguration; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.io.JobDirectoryFileSystem; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import jakarta.servlet.http.HttpServletResponse; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import org.apache.spark.sql.SparkSession; +import org.apache.hadoop.conf.Configuration; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Parameters; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; @@ -57,6 +61,8 @@ class JobProviderTest { private MockHttpServletRequest request; private MockHttpServletResponse response; + @TempDir Path tempDir; + @BeforeEach void setUp() { jobRegistry = new JobRegistry(); @@ -70,8 +76,9 @@ void setUp() { when(asyncConfig.getCacheMaxAge()).thenReturn(60); when(config.getAsync()).thenReturn(asyncConfig); - final SparkSession spark = mock(SparkSession.class); - jobProvider = new JobProvider(config, jobRegistry, spark, "/tmp/test"); + final JobDirectoryFileSystem jobDirectoryFileSystem = + new JobDirectoryFileSystem(tempDir.toUri(), new Configuration()); + jobProvider = new JobProvider(config, jobRegistry, jobDirectoryFileSystem); request = new MockHttpServletRequest(); request.setMethod("GET"); // Set the servlet path to match the FHIR server mount point. @@ -279,4 +286,26 @@ void errorUnwrappingHandlesNestedCause() { .isInstanceOf(InvalidRequestException.class) .hasMessageContaining("Nested error message"); } + + @Test + void deleteJobFilesRemovesDirectoryFromJobDirectoryFileSystem() throws Exception { + // Regression test for issue #2612: deleteJobFiles must resolve the file system from the + // warehouse URI rather than the Hadoop default file system, otherwise non-local schemes such + // as s3a:// fail with "Wrong FS". + final Path jobsDir = tempDir.resolve("jobs").resolve(JOB_ID); + Files.createDirectories(jobsDir); + Files.writeString(jobsDir.resolve("output.ndjson"), "{}"); + assertThat(Files.exists(jobsDir)).isTrue(); + + jobProvider.deleteJobFiles(JOB_ID); + + assertThat(Files.exists(jobsDir)).isFalse(); + } + + @Test + void deleteJobFilesSucceedsWhenDirectoryDoesNotExist() throws Exception { + // Deleting a non-existent job directory should not throw; the underlying delete returns false + // and is logged as a warning. + jobProvider.deleteJobFiles(JOB_ID); + } } diff --git a/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java b/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java index 0a85de4cba..71ccfd8e72 100644 --- a/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java @@ -24,6 +24,7 @@ import au.csiro.pathling.async.StageMap; import au.csiro.pathling.cache.CacheableDatabase; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.io.JobDirectoryFileSystem; import au.csiro.pathling.library.PathlingContext; import au.csiro.pathling.library.io.source.DataSourceBuilder; import au.csiro.pathling.library.io.source.QueryableDataSource; @@ -170,9 +171,8 @@ public RequestTagFactory requestTagFactory( public JobProvider jobProvider( ServerConfiguration serverConfiguration, JobRegistry jobRegistry, - SparkSession sparkSession, - @Value("${pathling.storage.warehouseUrl}") String warehouseUrl) { - return new JobProvider(serverConfiguration, jobRegistry, sparkSession, warehouseUrl); + JobDirectoryFileSystem jobDirectoryFileSystem) { + return new JobProvider(serverConfiguration, jobRegistry, jobDirectoryFileSystem); } // NOTE: Removed @ConfigurationProperties to avoid duplicate bean registration From df6b8d6f5a91fe23ea162bec597a8fb3fd94e5df Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 13 May 2026 15:56:58 +1000 Subject: [PATCH 03/12] fix: Register JobDirectoryFileSystem bean in test configuration The unit-test Spring context does not component-scan the server's io package, so the JobDirectoryFileSystem dependency required by the test jobProvider bean could not be resolved. This caused every unit test importing FhirServerTestConfiguration to fail context loading. Provide an explicit JobDirectoryFileSystem bean in the test configuration, built from the running SparkSession and the existing warehouse and database name properties. --- .../pathling/util/FhirServerTestConfiguration.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java b/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java index 71ccfd8e72..c8419e6998 100644 --- a/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java +++ b/server/src/test/java/au/csiro/pathling/util/FhirServerTestConfiguration.java @@ -34,6 +34,7 @@ import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import jakarta.annotation.Nonnull; +import java.net.URI; import java.nio.file.Path; import org.apache.spark.sql.SparkSession; import org.springframework.beans.factory.annotation.Value; @@ -175,6 +176,18 @@ public JobProvider jobProvider( return new JobProvider(serverConfiguration, jobRegistry, jobDirectoryFileSystem); } + @Bean + @Primary + @ConditionalOnMissingBean + public JobDirectoryFileSystem jobDirectoryFileSystem( + SparkSession sparkSession, + @Value("${pathling.storage.warehouseUrl}") String warehouseUrl, + @Value("${pathling.storage.databaseName}") String databaseName) { + return new JobDirectoryFileSystem( + URI.create(warehouseUrl + "/" + databaseName), + sparkSession.sparkContext().hadoopConfiguration()); + } + // NOTE: Removed @ConfigurationProperties to avoid duplicate bean registration // from @Bean methods inside ServerConfiguration class. // Tests should set required properties directly or use @TestPropertySource. From 7a26e8638c3ca9d03bfdb99c2560bb6c36b31b75 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 09:56:28 +1000 Subject: [PATCH 04/12] chore: Suppress new Trivy CVEs and upgrade mermaid to 11.15.0 Added suppressions for newly reported CVEs across core libraries, server, and site scopes following contextual impact assessment. All suppressed findings are either not bundled in the distribution or have unreachable vulnerable code paths. Upgraded mermaid from 11.12.2 to 11.15.0 via package.json override to fix four MEDIUM CVEs (CSS/HTML injection and DoS in diagram rendering) in the deployed static site. Co-Authored-By: Claude Sonnet 4.6 --- .trivyignore | 20 ++++++++++++++++++++ server/.trivyignore | 6 ++++++ site/.trivyignore | 8 ++++++++ site/bun.lock | 39 +++++++++------------------------------ site/package.json | 3 ++- 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/.trivyignore b/.trivyignore index 174e1b4dee..8505591a08 100644 --- a/.trivyignore +++ b/.trivyignore @@ -10,6 +10,14 @@ CVE-2025-58056 CVE-2025-67735 CVE-2026-33870 CVE-2026-33871 +CVE-2026-42583 +CVE-2026-42579 +CVE-2026-42584 +CVE-2026-42587 +CVE-2026-41417 +CVE-2026-42580 +CVE-2026-42581 +CVE-2026-42585 # The vulnerable version of protobuf-java is a transitive provided dependency, we do not bundle it into our distribution. CVE-2024-7254 @@ -53,3 +61,15 @@ CVE-2025-67721 # jackson-core async parser DoS — Pathling uses only synchronous parsing via HAPI FHIR. GHSA-72hv-8253-57qq + +# Apache Thrift TSSLTransportFactory certificate hostname validation flaw, no fixed version +# available. libthrift is a transitive dependency via hapi-fhir-structures-r4 -> jena-shex -> +# jena-arq. Pathling does not use Thrift's SSL transport, so the vulnerable code path is +# unreachable. +CVE-2026-43869 + +# OpenTelemetry W3C Baggage propagation unbounded memory allocation — opentelemetry-api is a +# transitive compile-scope dependency via hapi-fhir-base. In the Spark library context, no +# HTTP request processing is performed and OTel propagators are not configured, so the +# vulnerable Baggage parsing code path is never reached. +CVE-2026-45292 diff --git a/server/.trivyignore b/server/.trivyignore index 6508758c9d..1ffa14ece8 100644 --- a/server/.trivyignore +++ b/server/.trivyignore @@ -82,3 +82,9 @@ CVE-2025-67721 # Thrift's SSL transport (no TSSL/TSocket/Thrift client code anywhere in the # server), so the vulnerable code path is unreachable. CVE-2026-43869 + +# OpenTelemetry W3C Baggage propagation unbounded memory allocation — opentelemetry-api is a +# transitive dependency via hapi-fhir-base. The server does not configure the OTel SDK or +# W3C Baggage propagators; the API jar is present only for HAPI FHIR instrumentation +# annotations, so the vulnerable Baggage parsing code path is never reached. +CVE-2026-45292 diff --git a/site/.trivyignore b/site/.trivyignore index 22aaf62354..3b7a2cf90e 100644 --- a/site/.trivyignore +++ b/site/.trivyignore @@ -46,3 +46,11 @@ CVE-2026-41305 # uuid out-of-bounds write — build-time Docusaurus internal use only, not deployed. CVE-2026-41907 + +# @babel/plugin-transform-modules-systemjs arbitrary code generation — triggered only by +# malicious source files fed to the build-time Babel transpiler. Not deployed in the static site. +CVE-2026-44728 + +# fast-uri normalize() percent-encoded authority delimiter issue — fast-uri is pulled in by +# ajv, a build-time JSON schema validator used by Docusaurus tooling. Not deployed. +CVE-2026-6322 diff --git a/site/bun.lock b/site/bun.lock index 42dd90ca1a..349fb3c9a7 100644 --- a/site/bun.lock +++ b/site/bun.lock @@ -23,6 +23,7 @@ "lodash": "4.18.0", "lodash-es": "4.18.0", "mdast-util-to-hast": "13.2.1", + "mermaid": "^11.15.0", "node-forge": "1.3.3", "qs": "6.15.0", }, @@ -273,15 +274,7 @@ "@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.2", "", {}, "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA=="], - "@chevrotain/cst-dts-gen": ["@chevrotain/cst-dts-gen@11.0.3", "", { "dependencies": { "@chevrotain/gast": "11.0.3", "@chevrotain/types": "11.0.3", "lodash-es": "4.17.21" } }, "sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ=="], - - "@chevrotain/gast": ["@chevrotain/gast@11.0.3", "", { "dependencies": { "@chevrotain/types": "11.0.3", "lodash-es": "4.17.21" } }, "sha512-+qNfcoNk70PyS/uxmj3li5NiECO+2YKZZQMbmjTqRI3Qchu8Hig/Q9vgkHpI3alNjr7M+a2St5pw5w5F6NL5/Q=="], - - "@chevrotain/regexp-to-ast": ["@chevrotain/regexp-to-ast@11.0.3", "", {}, "sha512-1fMHaBZxLFvWI067AVbGJav1eRY7N8DDvYCTwGBiE/ytKBgP8azTdgyrKyWZ9Mfh09eHWb5PgTSO8wi7U824RA=="], - - "@chevrotain/types": ["@chevrotain/types@11.0.3", "", {}, "sha512-gsiM3G8b58kZC2HaWR50gu6Y1440cHiJ+i3JUvcp/35JchYejb2+5MVeJK0iKThYpAa/P2PYFV4hoi44HD+aHQ=="], - - "@chevrotain/utils": ["@chevrotain/utils@11.0.3", "", {}, "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ=="], + "@chevrotain/types": ["@chevrotain/types@11.1.2", "", {}, "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw=="], "@colors/colors": ["@colors/colors@1.5.0", "", {}, "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="], @@ -505,7 +498,7 @@ "@mdx-js/react": ["@mdx-js/react@3.1.1", "", { "dependencies": { "@types/mdx": "^2.0.0" }, "peerDependencies": { "@types/react": ">=16", "react": ">=16" } }, "sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw=="], - "@mermaid-js/parser": ["@mermaid-js/parser@0.6.3", "", { "dependencies": { "langium": "3.3.1" } }, "sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA=="], + "@mermaid-js/parser": ["@mermaid-js/parser@1.1.1", "", { "dependencies": { "@chevrotain/types": "~11.1.1" } }, "sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw=="], "@noble/hashes": ["@noble/hashes@1.4.0", "", {}, "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg=="], @@ -747,6 +740,8 @@ "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], + "@upsetjs/venn.js": ["@upsetjs/venn.js@2.0.0", "", { "optionalDependencies": { "d3-selection": "^3.0.0", "d3-transition": "^3.0.1" } }, "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw=="], + "@webassemblyjs/ast": ["@webassemblyjs/ast@1.14.1", "", { "dependencies": { "@webassemblyjs/helper-numbers": "1.13.2", "@webassemblyjs/helper-wasm-bytecode": "1.13.2" } }, "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ=="], "@webassemblyjs/floating-point-hex-parser": ["@webassemblyjs/floating-point-hex-parser@1.13.2", "", {}, "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA=="], @@ -915,10 +910,6 @@ "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], - "chevrotain": ["chevrotain@11.0.3", "", { "dependencies": { "@chevrotain/cst-dts-gen": "11.0.3", "@chevrotain/gast": "11.0.3", "@chevrotain/regexp-to-ast": "11.0.3", "@chevrotain/types": "11.0.3", "@chevrotain/utils": "11.0.3", "lodash-es": "4.17.21" } }, "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw=="], - - "chevrotain-allstar": ["chevrotain-allstar@0.3.1", "", { "dependencies": { "lodash-es": "^4.17.21" }, "peerDependencies": { "chevrotain": "^11.0.0" } }, "sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw=="], - "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], "chrome-trace-event": ["chrome-trace-event@1.0.4", "", {}, "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ=="], @@ -1105,7 +1096,7 @@ "d3-zoom": ["d3-zoom@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "2 - 3", "d3-transition": "2 - 3" } }, "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw=="], - "dagre-d3-es": ["dagre-d3-es@7.0.13", "", { "dependencies": { "d3": "^7.9.0", "lodash-es": "^4.17.21" } }, "sha512-efEhnxpSuwpYOKRm/L5KbqoZmNNukHa/Flty4Wp62JRvgH2ojwVgPgdYyr4twpieZnyRDdIH7PY2mopX26+j2Q=="], + "dagre-d3-es": ["dagre-d3-es@7.0.14", "", { "dependencies": { "d3": "^7.9.0", "lodash-es": "^4.17.21" } }, "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg=="], "dayjs": ["dayjs@1.11.19", "", {}, "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw=="], @@ -1203,6 +1194,8 @@ "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + "es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="], + "esast-util-from-estree": ["esast-util-from-estree@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "unist-util-position-from-estree": "^2.0.0" } }, "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ=="], "esast-util-from-js": ["esast-util-from-js@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "acorn": "^8.0.0", "esast-util-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw=="], @@ -1555,8 +1548,6 @@ "kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], - "langium": ["langium@3.3.1", "", { "dependencies": { "chevrotain": "~11.0.3", "chevrotain-allstar": "~0.3.0", "vscode-languageserver": "~9.0.1", "vscode-languageserver-textdocument": "~1.0.11", "vscode-uri": "~3.0.8" } }, "sha512-QJv/h939gDpvT+9SiLVlY7tZC3xB2qK57v0J04Sh9wpMb6MP1q8gB21L3WIo8T5P1MSMg3Ep14L7KkDCFG3y4w=="], - "latest-version": ["latest-version@7.0.0", "", { "dependencies": { "package-json": "^8.1.0" } }, "sha512-KvNT4XqAMzdcL6ka6Tl3i2lYeFDgXNCuIX+xNx6ZMVR1dFq+idXd9FLKNMOIx0t9mJ9/HudyX4oZWXZQ0UJHeg=="], "launch-editor": ["launch-editor@2.12.0", "", { "dependencies": { "picocolors": "^1.1.1", "shell-quote": "^1.8.3" } }, "sha512-giOHXoOtifjdHqUamwKq6c49GzBdLjvxrd2D+Q4V6uOHopJv7p9VJxikDsQ/CBXZbEITgUqSVHXLTG3VhPP1Dg=="], @@ -1651,7 +1642,7 @@ "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], - "mermaid": ["mermaid@11.12.2", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.1", "@mermaid-js/parser": "^0.6.3", "@types/d3": "^7.4.3", "cytoscape": "^3.29.3", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.13", "dayjs": "^1.11.18", "dompurify": "^3.2.5", "katex": "^0.16.22", "khroma": "^2.1.0", "lodash-es": "^4.17.21", "marked": "^16.2.1", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0" } }, "sha512-n34QPDPEKmaeCG4WDMGy0OT6PSyxKCfy2pJgShP+Qow2KLrvWjclwbc3yXfSIf4BanqWEhQEpngWwNp/XhZt6w=="], + "mermaid": ["mermaid@11.15.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.1.1", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.1", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.19", "dompurify": "^3.3.1", "es-toolkit": "^1.45.1", "katex": "^0.16.25", "khroma": "^2.1.0", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" } }, "sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw=="], "methods": ["methods@1.1.2", "", {}, "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="], @@ -2391,18 +2382,6 @@ "vfile-message": ["vfile-message@4.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="], - "vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="], - - "vscode-languageserver": ["vscode-languageserver@9.0.1", "", { "dependencies": { "vscode-languageserver-protocol": "3.17.5" }, "bin": { "installServerIntoExtension": "bin/installServerIntoExtension" } }, "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g=="], - - "vscode-languageserver-protocol": ["vscode-languageserver-protocol@3.17.5", "", { "dependencies": { "vscode-jsonrpc": "8.2.0", "vscode-languageserver-types": "3.17.5" } }, "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg=="], - - "vscode-languageserver-textdocument": ["vscode-languageserver-textdocument@1.0.12", "", {}, "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA=="], - - "vscode-languageserver-types": ["vscode-languageserver-types@3.17.5", "", {}, "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg=="], - - "vscode-uri": ["vscode-uri@3.0.8", "", {}, "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw=="], - "watchpack": ["watchpack@2.5.1", "", { "dependencies": { "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" } }, "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg=="], "wbuf": ["wbuf@1.7.3", "", { "dependencies": { "minimalistic-assert": "^1.0.0" } }, "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA=="], diff --git a/site/package.json b/site/package.json index 7e57055c6b..8638d5a740 100644 --- a/site/package.json +++ b/site/package.json @@ -44,6 +44,7 @@ "node-forge": "1.3.3", "qs": "6.15.0", "js-yaml": "3.14.2", - "mdast-util-to-hast": "13.2.1" + "mdast-util-to-hast": "13.2.1", + "mermaid": "^11.15.0" } } From 26d2e4623f57ce1acf957ab5fdfd2d37b94f5652 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 13:17:39 +1000 Subject: [PATCH 05/12] fix: Prevent standalone Spark tests from destroying shared SparkContext Three unit tests (SqlQueryResultStreamerTest, ViewRegistrationServiceTest, LibraryReferenceResolverTest.CanonicalReferences) called spark.stop() in @AfterAll, which destroyed the JVM-wide SparkContext and caused ViewDefinitionSearchTest and ViewDefinitionCreateTest to fail intermittently depending on test execution order. Converted all three tests to @SpringBootUnitTest so they receive the shared SparkSession via Spring injection, consistent with every other Spark-dependent test in the server module. The manually created sessions and @AfterAll teardowns are removed entirely. Closes #2615 Co-Authored-By: Claude Sonnet 4.6 --- .../LibraryReferenceResolverTest.java | 37 ++++-------------- .../sqlquery/SqlQueryResultStreamerTest.java | 38 ++++--------------- .../sqlquery/ViewRegistrationServiceTest.java | 33 +++++----------- 3 files changed, 24 insertions(+), 84 deletions(-) diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java index 1c8e19e2a6..426cbabfda 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/LibraryReferenceResolverTest.java @@ -28,6 +28,7 @@ import au.csiro.pathling.errors.ResourceNotFoundError; import au.csiro.pathling.io.source.DataSource; import au.csiro.pathling.read.ReadExecutor; +import au.csiro.pathling.test.SpringBootUnitTest; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import java.util.List; @@ -38,21 +39,18 @@ import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; import org.hl7.fhir.r4.model.Library; import org.hl7.fhir.r4.model.Reference; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.beans.factory.annotation.Autowired; /** * Tests for {@link LibraryReferenceResolver} covering both the relative-literal and canonical * reference resolution paths. */ -@TestInstance(Lifecycle.PER_CLASS) +@SpringBootUnitTest class LibraryReferenceResolverTest { // --------------------------------------------------------------------------- @@ -127,39 +125,18 @@ void translatesNoDataIllegalArgumentToResourceNotFoundException() { } // --------------------------------------------------------------------------- - // Canonical references — uses a real Spark session + FhirEncoders. + // Canonical references — uses the shared Spark session and FhirEncoders. // --------------------------------------------------------------------------- @Nested - @TestInstance(Lifecycle.PER_CLASS) class CanonicalReferences { - private SparkSession spark; - private FhirEncoders fhirEncoders; + @Autowired private SparkSession spark; + @Autowired private FhirEncoders fhirEncoders; + private DataSource dataSource; private LibraryReferenceResolver resolver; - @BeforeAll - void setUpAll() { - spark = - SparkSession.builder() - .master("local[1]") - .appName("LibraryReferenceResolverTest") - .config("spark.driver.bindAddress", "localhost") - .config("spark.driver.host", "localhost") - .config("spark.ui.enabled", false) - .config("spark.sql.shuffle.partitions", 1) - .getOrCreate(); - fhirEncoders = FhirEncoders.forR4().getOrCreate(); - } - - @AfterAll - void tearDownAll() { - if (spark != null) { - spark.stop(); - } - } - @BeforeEach void setUp() { dataSource = mock(DataSource.class); diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryResultStreamerTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryResultStreamerTest.java index be5d7404cc..d4d3b70985 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryResultStreamerTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryResultStreamerTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import au.csiro.pathling.test.SpringBootUnitTest; import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.spark.sql.Dataset; @@ -27,44 +28,21 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.mock.web.MockHttpServletResponse; /** - * Tests for {@link SqlQueryResultStreamer} covering each output format. Uses a real local - * SparkSession to materialise a small Dataset and a Spring {@link MockHttpServletResponse} to - * capture written bytes and headers. + * Tests for {@link SqlQueryResultStreamer} covering each output format. Uses the shared Spark + * session to materialise a small Dataset and a Spring {@link MockHttpServletResponse} to capture + * written bytes and headers. */ -@TestInstance(Lifecycle.PER_CLASS) +@SpringBootUnitTest class SqlQueryResultStreamerTest { - private SparkSession spark; - private SqlQueryResultStreamer streamer; - - @BeforeAll - void setUpAll() { - spark = - SparkSession.builder() - .master("local[1]") - .appName("SqlQueryResultStreamerTest") - .config("spark.driver.bindAddress", "localhost") - .config("spark.driver.host", "localhost") - .config("spark.ui.enabled", false) - .config("spark.sql.shuffle.partitions", 1) - .getOrCreate(); - streamer = new SqlQueryResultStreamer(); - } + @Autowired private SparkSession spark; - @AfterAll - void tearDownAll() { - if (spark != null) { - spark.stop(); - } - } + private final SqlQueryResultStreamer streamer = new SqlQueryResultStreamer(); @Test void streamsNdjsonWithUtf8Encoding() { diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java index e81780bbf6..c42e3985ba 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/ViewRegistrationServiceTest.java @@ -20,6 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat; import au.csiro.pathling.config.ServerConfiguration; +import au.csiro.pathling.test.SpringBootUnitTest; import ca.uhn.fhir.context.FhirContext; import java.util.Arrays; import java.util.List; @@ -35,42 +36,26 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.springframework.beans.factory.annotation.Autowired; /** * Tests for {@link ViewRegistrationService}, with particular attention to the request-id * namespacing that prevents concurrent {@code $sqlquery-run} requests from clobbering one another's * temporary views in Spark's session-global catalog. */ -@TestInstance(Lifecycle.PER_CLASS) +@SpringBootUnitTest class ViewRegistrationServiceTest { - private SparkSession spark; + @Autowired private SparkSession spark; + @Autowired private FhirContext fhirContext; + private ViewRegistrationService service; - @BeforeAll + @BeforeEach void setUp() { - spark = - SparkSession.builder() - .master("local[2]") - .appName("ViewRegistrationServiceTest") - .config("spark.driver.bindAddress", "localhost") - .config("spark.driver.host", "localhost") - .config("spark.ui.enabled", false) - .config("spark.sql.shuffle.partitions", 1) - .getOrCreate(); - service = new ViewRegistrationService(spark, FhirContext.forR4(), new ServerConfiguration()); - } - - @AfterAll - void tearDown() { - if (spark != null) { - spark.stop(); - } + service = new ViewRegistrationService(spark, fhirContext, new ServerConfiguration()); } // --------------------------------------------------------------------------- From 006d056be9b55bb0cc6a11dc53e0515ac01f4895 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 13:35:30 +1000 Subject: [PATCH 06/12] fix: Clear SecurityContext after each test in BulkSubmitProviderTest BulkSubmitProviderTest was installing a Mockito mock as the active Spring SecurityContext in @BeforeEach but never clearing it. Under JUnit 5 parallel execution the mock leaked onto adjacent threads, causing SearchProviderAuthTest to inherit a mock context in which setAuthentication() is a no-op, so checkHasAuthority() would throw "Token not present". Closes #2617. Co-Authored-By: Claude Sonnet 4.6 --- .../operations/bulksubmit/BulkSubmitProviderTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitProviderTest.java b/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitProviderTest.java index 05f2e8fd37..13edb97344 100644 --- a/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitProviderTest.java +++ b/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitProviderTest.java @@ -34,6 +34,7 @@ import java.util.Optional; import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.security.core.Authentication; @@ -81,6 +82,11 @@ void setUp() { SecurityContextHolder.setContext(securityContext); } + @AfterEach + void tearDown() { + SecurityContextHolder.clearContext(); + } + // ======================================== // In-Progress Submission Tests // ======================================== From 906ef4933ed89a2e582e4d9a7402e93f38ad0b1b Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 17:01:04 +1000 Subject: [PATCH 07/12] fix: Match staged-file URIs in $import-pnp allowlist Hadoop Path.toUri() drops the empty authority on file:// paths built via new Path(parent, child), yielding file:/path. Files discovered later via fs.listFiles + fs.makeQualified preserve the empty authority and come back as file:///path. UrlAllowlist's string-prefix match then rejects the downloaded file URLs against the staging-directory prefix, failing the import with an AccessDeniedError after the bulk export has already completed. Build the prefix via fs.getFileStatus so both sides use the same canonical URI form. Co-Authored-By: Claude Sonnet 4.6 --- .../operations/bulkimport/ImportPnpExecutor.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/au/csiro/pathling/operations/bulkimport/ImportPnpExecutor.java b/server/src/main/java/au/csiro/pathling/operations/bulkimport/ImportPnpExecutor.java index 98d95b1da2..069f27a970 100644 --- a/server/src/main/java/au/csiro/pathling/operations/bulkimport/ImportPnpExecutor.java +++ b/server/src/main/java/au/csiro/pathling/operations/bulkimport/ImportPnpExecutor.java @@ -158,9 +158,12 @@ public ImportResponse execute(@Nonnull final ImportPnpRequest pnpRequest, final // Execute the import using the existing ImportExecutor with custom allowable sources. // This bypasses the configured allowableSources validation for the staging directory, - // which the server downloaded and trusts. The qualified URI ensures the prefix matches - // whatever scheme the staging file system uses. - final List pnpAllowableSources = List.of(tempDir.toUri().toString() + "/"); + // which the server downloaded and trusts. Go via fs.getFileStatus() to obtain a URI in + // the same canonical form (with empty authority preserved on file://) that fs.listFiles + // produces for the downloaded files, so the UrlAllowlist string-prefix match holds: + // tempDir.toUri() alone yields file:/path while listed files come back as file:///path. + final List pnpAllowableSources = + List.of(fs.getFileStatus(tempDir).getPath().toUri().toString() + "/"); final ImportResponse response = importExecutor.execute(importRequest, jobId, pnpAllowableSources); From ca1ce49c07e0e5f7cb5d27e89813c359209588cc Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 17:01:23 +1000 Subject: [PATCH 08/12] fix: Clear Delta cache between $import-pnp integration tests The shared static warehouse @TempDir is cleaned in @AfterEach, but Spark's catalog cache and Delta's global DeltaLog cache still hold references to the deleted tables. The next test rebuilds the warehouse from test fixtures, but isDeltaTable returns false against the stale log, so the import falls through to an ERROR_IF_EXISTS write that collides with the freshly-copied directory and fails with DELTA_PATH_EXISTS. Clear both caches before deleting files so cleanup restores both the on-disk and in-memory state. Co-Authored-By: Claude Sonnet 4.6 --- .../pathling/operations/bulkimport/ImportPnpOperationIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java b/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java index b20a3e3261..0f1075ff0c 100644 --- a/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java @@ -173,6 +173,11 @@ void setup() { @AfterEach void cleanup() throws IOException { + // Clear cached Delta table state before deleting files. Otherwise the next test sees a stale + // DeltaLog in memory that no longer matches the on-disk warehouse rebuilt from test fixtures, + // and Delta refuses the import with DELTA_PATH_EXISTS. + pathlingContext.getSpark().catalog().clearCache(); + org.apache.spark.sql.delta.DeltaLog.clearCache(); FileUtils.cleanDirectory(warehouseDir.toFile()); } From 780db513ed9a060686510d05f6d5b27f658672d6 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 17:01:31 +1000 Subject: [PATCH 09/12] fix: Send auth token in poisoned-manifest exfiltration test The test runs under the integration-test profile with PNP credentials configured and auth enabled, but its requests were missing the Authorization header. The pre-existing 401 was hidden by an earlier PNP allowlist bug; with that fix in place, the auth interlock now rejects the request before the poisoning scenario can exercise. Co-Authored-By: Claude Sonnet 4.6 --- .../pathling/operations/bulkimport/ImportPnpOperationIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java b/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java index 0f1075ff0c..ff331f9b82 100644 --- a/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/bulkimport/ImportPnpOperationIT.java @@ -701,6 +701,7 @@ void testPoisonedManifestTypeFailsJobAndBlocksExfiltration() throws IOException .header("Content-Type", "application/fhir+json") .header("Accept", "application/fhir+json") .header("Prefer", "respond-async") + .header("Authorization", "Bearer " + AUTH_TOKEN) .bodyValue(requestBody) .exchange() .expectStatus() @@ -736,6 +737,7 @@ void testPoisonedManifestTypeFailsJobAndBlocksExfiltration() throws IOException .get() .uri(contentLocation) .header("Accept", "application/fhir+json") + .header("Authorization", "Bearer " + AUTH_TOKEN) .exchange() .expectStatus() .is4xxClientError() @@ -752,6 +754,7 @@ void testPoisonedManifestTypeFailsJobAndBlocksExfiltration() throws IOException webTestClient .get() .uri("http://localhost:" + port + "/jobs/" + jobId + "/escaped.0000.ndjson") + .header("Authorization", "Bearer " + AUTH_TOKEN) .exchange() .expectStatus() .isNotFound(); From aa4fdc6862032aea1776c9b018437d2f7d946557 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 17:01:39 +1000 Subject: [PATCH 10/12] fix: Include WireMock port in $bulk-submit IT allowable sources The integration tests configured pathling.bulk-submit.allowable-sources to bare http://localhost via @TestPropertySource. The URI-aware UrlAllowlist resolves that prefix to effective port 80 and no longer matches the dynamic http://localhost:{wireMockPort} the tests actually use. Move the property into @DynamicPropertySource so it picks up the WireMock port at runtime. Co-Authored-By: Claude Sonnet 4.6 --- .../pathling/operations/bulksubmit/BulkSubmitOAuthIT.java | 6 +++++- .../operations/bulksubmit/BulkSubmitOperationIT.java | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOAuthIT.java b/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOAuthIT.java index 6b0e3b38e6..a066e582ca 100644 --- a/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOAuthIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOAuthIT.java @@ -71,7 +71,6 @@ @TestPropertySource( properties = { "pathling.async.enabled=true", - "pathling.bulk-submit.allowable-sources[0]=http://localhost", // Configure submitter with OAuth credentials for symmetric (client_secret) auth. "pathling.bulk-submit.allowed-submitters[0].system=http://example.org/submitters", "pathling.bulk-submit.allowed-submitters[0].value=oauth-submitter", @@ -118,6 +117,11 @@ static void configureProperties(final DynamicPropertyRegistry registry) { TestDataSetup.copyTestDataToTempDir(warehouseDir, "Condition"); registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir.toAbsolutePath()); registry.add("pathling.bulk-submit.staging-directory", stagingDir::toString); + // The allowable source must include the WireMock port. Bare "http://localhost" (port 80) + // no longer matches "http://localhost:{port}" under the URI-aware UrlAllowlist matching. + registry.add( + "pathling.bulk-submit.allowable-sources[0]", + () -> "http://localhost:" + wireMockServer.port()); } @BeforeEach diff --git a/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOperationIT.java b/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOperationIT.java index 656af2200b..d8ebbe33ea 100644 --- a/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOperationIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/bulksubmit/BulkSubmitOperationIT.java @@ -67,7 +67,6 @@ @TestPropertySource( properties = { "pathling.async.enabled=true", - "pathling.bulk-submit.allowable-sources[0]=http://localhost", "pathling.bulk-submit.allowed-submitters[0].system=http://example.org/submitters", "pathling.bulk-submit.allowed-submitters[0].value=test-submitter" }) @@ -107,6 +106,11 @@ static void configureProperties(final DynamicPropertyRegistry registry) { TestDataSetup.copyTestDataToTempDir(warehouseDir, "Condition"); registry.add("pathling.storage.warehouseUrl", () -> "file://" + warehouseDir.toAbsolutePath()); registry.add("pathling.bulk-submit.staging-directory", stagingDir::toString); + // The allowable source must include the WireMock port. Bare "http://localhost" (port 80) + // no longer matches "http://localhost:{port}" under the URI-aware UrlAllowlist matching. + registry.add( + "pathling.bulk-submit.allowable-sources[0]", + () -> "http://localhost:" + wireMockServer.port()); } @BeforeEach From e1ad55baea97421ea05cd7cc92fc54ac41f32f2e Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 19 May 2026 18:16:38 +1000 Subject: [PATCH 11/12] fix: Set 60s response timeout on SqlQueryRunDeltaIT WebTestClient The test relied on the WebTestClient default response timeout of 5 s, which is shorter than the cold-start latency of the first POST against a freshly started Spring Boot context with a Delta-backed warehouse. Match the 60 s timeout already used by the sibling integration tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java index 7769e11e94..d926e01b50 100644 --- a/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java +++ b/server/src/test/java/au/csiro/pathling/operations/sqlquery/SqlQueryRunDeltaIT.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -106,6 +107,7 @@ void setup() { webTestClient .mutate() .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(100 * 1024 * 1024)) + .responseTimeout(Duration.ofSeconds(60)) .build(); jsonParser = fhirContext.newJsonParser(); } From 19f563ff914086430bfc108a2ecfa1185bcad66d Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sun, 17 May 2026 10:54:57 +0200 Subject: [PATCH 12/12] Update OpenSpec files --- .claude/commands/opsx-apply.md | 152 +++++ .claude/commands/opsx-archive.md | 156 +++++ .claude/commands/opsx-bulk-archive.md | 245 ++++++++ .claude/commands/opsx-continue.md | 116 ++++ .claude/commands/opsx-explore.md | 178 ++++++ .claude/commands/opsx-ff.md | 101 +++ .claude/commands/opsx-new.md | 75 +++ .claude/commands/opsx-onboard.md | 567 +++++++++++++++++ .claude/commands/opsx-sync.md | 137 +++++ .claude/commands/opsx-verify.md | 164 +++++ .claude/commands/opsx/apply.md | 13 +- .claude/commands/opsx/archive.md | 11 +- .claude/commands/opsx/bulk-archive.md | 14 +- .claude/commands/opsx/continue.md | 5 - .claude/commands/opsx/explore.md | 18 +- .claude/commands/opsx/ff.md | 9 +- .claude/commands/opsx/new.md | 1 - .claude/commands/opsx/onboard.md | 77 ++- .claude/commands/opsx/sync.md | 7 - .claude/commands/opsx/verify.md | 18 +- .claude/skills/openspec-apply-change/SKILL.md | 17 +- .../skills/openspec-archive-change/SKILL.md | 11 +- .../openspec-bulk-archive-change/SKILL.md | 16 +- .../skills/openspec-continue-change/SKILL.md | 7 +- .claude/skills/openspec-explore/SKILL.md | 27 +- .claude/skills/openspec-ff-change/SKILL.md | 8 +- .claude/skills/openspec-new-change/SKILL.md | 3 +- .claude/skills/openspec-onboard/SKILL.md | 85 ++- .claude/skills/openspec-sync-specs/SKILL.md | 9 +- .../skills/openspec-verify-change/SKILL.md | 20 +- .pi/prompts/opsx-apply.md | 153 +++++ .pi/prompts/opsx-archive.md | 157 +++++ .pi/prompts/opsx-bulk-archive.md | 246 ++++++++ .pi/prompts/opsx-continue.md | 117 ++++ .pi/prompts/opsx-explore.md | 179 ++++++ .pi/prompts/opsx-ff.md | 102 ++++ .pi/prompts/opsx-new.md | 76 +++ .pi/prompts/opsx-onboard.md | 567 +++++++++++++++++ .pi/prompts/opsx-sync.md | 138 +++++ .pi/prompts/opsx-verify.md | 165 +++++ .pi/skills/openspec-apply-change/SKILL.md | 159 +++++ .pi/skills/openspec-archive-change/SKILL.md | 116 ++++ .../openspec-bulk-archive-change/SKILL.md | 252 ++++++++ .pi/skills/openspec-continue-change/SKILL.md | 123 ++++ .pi/skills/openspec-explore/SKILL.md | 299 +++++++++ .pi/skills/openspec-ff-change/SKILL.md | 108 ++++ .pi/skills/openspec-new-change/SKILL.md | 83 +++ .pi/skills/openspec-onboard/SKILL.md | 574 ++++++++++++++++++ .pi/skills/openspec-sync-specs/SKILL.md | 144 +++++ .pi/skills/openspec-verify-change/SKILL.md | 171 ++++++ 50 files changed, 5975 insertions(+), 221 deletions(-) create mode 100644 .claude/commands/opsx-apply.md create mode 100644 .claude/commands/opsx-archive.md create mode 100644 .claude/commands/opsx-bulk-archive.md create mode 100644 .claude/commands/opsx-continue.md create mode 100644 .claude/commands/opsx-explore.md create mode 100644 .claude/commands/opsx-ff.md create mode 100644 .claude/commands/opsx-new.md create mode 100644 .claude/commands/opsx-onboard.md create mode 100644 .claude/commands/opsx-sync.md create mode 100644 .claude/commands/opsx-verify.md create mode 100644 .pi/prompts/opsx-apply.md create mode 100644 .pi/prompts/opsx-archive.md create mode 100644 .pi/prompts/opsx-bulk-archive.md create mode 100644 .pi/prompts/opsx-continue.md create mode 100644 .pi/prompts/opsx-explore.md create mode 100644 .pi/prompts/opsx-ff.md create mode 100644 .pi/prompts/opsx-new.md create mode 100644 .pi/prompts/opsx-onboard.md create mode 100644 .pi/prompts/opsx-sync.md create mode 100644 .pi/prompts/opsx-verify.md create mode 100644 .pi/skills/openspec-apply-change/SKILL.md create mode 100644 .pi/skills/openspec-archive-change/SKILL.md create mode 100644 .pi/skills/openspec-bulk-archive-change/SKILL.md create mode 100644 .pi/skills/openspec-continue-change/SKILL.md create mode 100644 .pi/skills/openspec-explore/SKILL.md create mode 100644 .pi/skills/openspec-ff-change/SKILL.md create mode 100644 .pi/skills/openspec-new-change/SKILL.md create mode 100644 .pi/skills/openspec-onboard/SKILL.md create mode 100644 .pi/skills/openspec-sync-specs/SKILL.md create mode 100644 .pi/skills/openspec-verify-change/SKILL.md diff --git a/.claude/commands/opsx-apply.md b/.claude/commands/opsx-apply.md new file mode 100644 index 0000000000..201edf0922 --- /dev/null +++ b/.claude/commands/opsx-apply.md @@ -0,0 +1,152 @@ +--- +description: Implement tasks from an OpenSpec change (Experimental) +--- + +Implement tasks from an OpenSpec change. + +**Input**: Optionally specify a change name (e.g., `/opsx-apply add-auth`). If omitted, check if it can be inferred from conversation context. If vague or ambiguous you MUST prompt for available changes. + +**Steps** + +1. **Select the change** + + If a name is provided, use it. Otherwise: + - Infer from conversation context if the user mentioned a change + - Auto-select if only one active change exists + - If ambiguous, run `openspec list --json` to get available changes and use the **AskUserQuestion tool** to let the user select + + Always announce: "Using change: " and how to override (e.g., `/opsx-apply `). + +2. **Check status to understand the schema** + + ```bash + openspec status --change "" --json + ``` + + Parse the JSON to understand: + - `schemaName`: The workflow being used (e.g., "spec-driven") + - Which artifact contains the tasks (typically "tasks" for spec-driven, check status for others) + +3. **Get apply instructions** + + ```bash + openspec instructions apply --change "" --json + ``` + + This returns: + - `contextFiles`: artifact ID -> array of concrete file paths (varies by schema) + - Progress (total, complete, remaining) + - Task list with status + - Dynamic instruction based on current state + + **Handle states:** + - If `state: "blocked"` (missing artifacts): show message, suggest using `/opsx-continue` + - If `state: "all_done"`: congratulate, suggest archive + - Otherwise: proceed to implementation + +4. **Read context files** + + Read every file path listed under `contextFiles` from the apply instructions output. + The files depend on the schema being used: + - **spec-driven**: proposal, specs, design, tasks + - Other schemas: follow the contextFiles from CLI output + +5. **Show current progress** + + Display: + - Schema being used + - Progress: "N/M tasks complete" + - Remaining tasks overview + - Dynamic instruction from CLI + +6. **Implement tasks (loop until done or blocked)** + + For each pending task: + - Show which task is being worked on + - Make the code changes required + - Keep changes minimal and focused + - Mark task complete in the tasks file: `- [ ]` → `- [x]` + - Continue to next task + + **Pause if:** + - Task is unclear → ask for clarification + - Implementation reveals a design issue → suggest updating artifacts + - Error or blocker encountered → report and wait for guidance + - User interrupts + +7. **On completion or pause, show status** + + Display: + - Tasks completed this session + - Overall progress: "N/M tasks complete" + - If all done: suggest archive + - If paused: explain why and wait for guidance + +**Output During Implementation** + +``` +## Implementing: (schema: ) + +Working on task 3/7: +[...implementation happening...] +✓ Task complete + +Working on task 4/7: +[...implementation happening...] +✓ Task complete +``` + +**Output On Completion** + +``` +## Implementation Complete + +**Change:** +**Schema:** +**Progress:** 7/7 tasks complete ✓ + +### Completed This Session +- [x] Task 1 +- [x] Task 2 +... + +All tasks complete! You can archive this change with `/opsx-archive`. +``` + +**Output On Pause (Issue Encountered)** + +``` +## Implementation Paused + +**Change:** +**Schema:** +**Progress:** 4/7 tasks complete + +### Issue Encountered + + +**Options:** +1.