From 58d85871527f507c6a349404513fbe61398ee9d6 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 7 Oct 2025 17:50:36 +1100 Subject: [PATCH 01/24] Fix bug in Dockerfile --- Dockerfile | 29 +++++++++---------- .../server/core/service/ElasticSearch.java | 1 + .../ogcapi/server/features/RestApiTest.java | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6ef41f5..21eabdcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,18 +4,17 @@ VOLUME /tmp ENV MAX_HEAP_PERCENTAGE=70 COPY ./server/target/ogcapi-java-server-*-exec.jar app.jar -ENTRYPOINT [\ - "java",\ - "-XX:MaxRAMPercentage=${MAX_HEAP_PERCENTAGE}",\ - "-Delasticsearch.index.name=${INDEX_NAME}",\ - "-Delasticsearch.cloud_optimized_index.name=${CO_INDEX_NAME}",\ - "-Delasticsearch.vocabs_index.name=${VOCABS_INDEX_NAME}",\ - "-Dapi.host=${HOST}:${PORT}",\ - "-Dserver.port=${PORT}",\ - "-Delasticsearch.serverUrl=${ELASTIC_URL}",\ - "-Delasticsearch.apiKey=${ELASTIC_KEY}",\ - "-Ddata-access-service.host=${DAS_HOST}",\ - "-Ddata-access-service.secret=${DAS_SECRET}",\ - "--enable-preview",\ - "-jar",\ - "/app.jar"] +ENTRYPOINT ["/bin/sh", "-c", "java \ + -XX:MaxRAMPercentage=${MAX_HEAP_PERCENTAGE} \ + -Delasticsearch.index.name=${INDEX_NAME} \ + -Delasticsearch.cloud_optimized_index.name=${CO_INDEX_NAME} \ + -Delasticsearch.vocabs_index.name=${VOCABS_INDEX_NAME} \ + -Dapi.host=${HOST}:${PORT} \ + -Dserver.port=${PORT} \ + -Delasticsearch.serverUrl=${ELASTIC_URL} \ + -Delasticsearch.apiKey=${ELASTIC_KEY} \ + -Ddata-access-service.host=${DAS_HOST} \ + -Ddata-access-service.secret=${DAS_SECRET} \ + --enable-preview \ + -jar \ + /app.jar"] diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 30d2bac2..a960fbea 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -315,6 +315,7 @@ public ElasticSearchBase.SearchResult searchByParameters(Li } } catch(Exception e) { + log.warn("Error parsing score assume null", e); // OK to ignore as accept null as the value } // Get the search after diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index beb47960..034b8425 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -320,12 +320,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "80", + "95", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:7709f541-fc0c-4318-b5b9-9053aa474e0e", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); From 80a8f3c3bd6f4a8f8ad7eef684e46902617981ac Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 7 Oct 2025 18:00:13 +1100 Subject: [PATCH 02/24] mvn do not enable test profile by default --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1dee1f2..e6ee4bce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: cache: 'maven' - name: Build with Maven - run: mvn -B verify --file pom.xml + run: mvn -B verify -P test --file pom.xml - name: Build Docker Image uses: docker/build-push-action@v5 From 3d5c056b683c481b080809bd8f451afa24dabb33 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 7 Oct 2025 18:04:33 +1100 Subject: [PATCH 03/24] rollback setting --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e6ee4bce..b1dee1f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: cache: 'maven' - name: Build with Maven - run: mvn -B verify -P test --file pom.xml + run: mvn -B verify --file pom.xml - name: Build Docker Image uses: docker/build-push-action@v5 From d54af5ad14638fec2a3c23d873236ae414f747b5 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 7 Oct 2025 18:14:44 +1100 Subject: [PATCH 04/24] clean test pack --- .../databag/8cdcdcad-399b-4bed-8cb2-29c486b6b124.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/test/resources/databag/8cdcdcad-399b-4bed-8cb2-29c486b6b124.json b/server/src/test/resources/databag/8cdcdcad-399b-4bed-8cb2-29c486b6b124.json index 007f0c29..be802a8f 100644 --- a/server/src/test/resources/databag/8cdcdcad-399b-4bed-8cb2-29c486b6b124.json +++ b/server/src/test/resources/databag/8cdcdcad-399b-4bed-8cb2-29c486b6b124.json @@ -1,6 +1,6 @@ { "title": "IMOS - National Reef Monitoring Network Sub-Facility", - "description": "The National Reef Monitoring Network brings together shallow reef surveys conducted around Australia into a centralised database. The IMOS National Reef Monitoring Network sub-Facility collates, cleans, stores and makes this data rapidly available from contributors including: Reef Life Survey, Parks Australia, Department of Biodiversity, Conservation and Attractions (Western Australia), Department of Environment, Water and Natural Resources (South Australia), Department of Primary Industries (New South Wales), Tasmanian Parks and Wildlife Service and Parks Victoria. The data provided by the National Reef Monitoring Network contributes to establishing and supporting national marine baselines, and assisting with the management of Commonwealth and State marine reserves. Reef Life Survey (RLS) and the Australian Temperate Reef Network (ATRC) aims to improve biodiversity conservation and the sustainable management of marine resources by coordinating surveys of rocky and coral reefs using scientific methods, with the ultimate goal to improve coastal stewardship. Our activities depend on the skills of marine scientists, experienced and motivated recreational SCUBA divers, partnerships with management agencies and university researchers, and active input from the ATRC partners and RLS Advisory Committee RLS and ATRC data are freely available to the public for non-profit purposes, so not only managers, but also groups such as local dive clubs or schools may use these data to look at changes over time in their own local reefs. By making data freely available and through public outputs, RLS and ATRC aims to raise broader community awareness of the status of Australia’s marine biodiversity and associated conservation issues.", + "description": "No value is fine, this case is make sure short form of National Reef Monitoring Network not appear here", "extent": { "bbox": [ [ @@ -40,7 +40,7 @@ }, "creation": "2021-05-21T12:00:00", "revision": "2024-07-04T03:50:26", - "ai:description": "The National Reef Monitoring Network brings together shallow reef surveys conducted around Australia into a centralised database. The IMOS National Reef Monitoring Network sub-Facility collates, cleans, stores and makes this data rapidly available from contributors including: - Reef Life Surve - Parks Australi - Department of Biodiversity, Conservation and Attractions (Western Australia) - Department of Environment, Water and Natural Resources (South Australia) - Department of Primary Industries (New South Wales) - Tasmanian Parks and Wildlife Service - Parks Victoria. The data provided by the National Reef Monitoring Network contributes to establishing and supporting national marine baselines, and assisting with the management of Commonwealth and State marine reserves. Reef Life Survey (RLS) and the Australian Temperate Reef Network (ATRC) aims to improve biodiversity conservation and the sustainable management of marine resources by coordinating surveys of rocky and coral reefs using scientific methods, with the ultimate goal to improve coastal stewardship. Our activities depend on the skills of marine scientists, experienced and motivated recreational SCUBA divers, partnerships with management agencies and university researchers, and active input from the ATRC partners and RLS Advisory Committee. RLS and ATRC data are freely available to the public for non-profit purposes, so not only managers, but also groups such as local dive clubs or schools may use these data to look at changes over time in their own local reefs. By making data freely available and through public outputs, RLS and ATRC aims to raise broader community awareness of the status of Australia’s marine biodiversity and associated conservation issues.", + "ai:description": "No value is fine", "dataset_provider": "IMOS", "update_frequency": "other", "proj:geometry": { @@ -11300,8 +11300,7 @@ "href": "uuid:0f65b7ae-1f6f-4a55-b804-1c991f791e1a", "rel": "sibling", "type": "application/json", - "title": "{\"title\":\"IMOS - Autonomous Underwater Vehicles - AUV Iver", - "recordAbstract": "The IMOS Autonomous Underwater Vehicles Facility has an Autonomous Underwater Vehicle (AUV) called Iver (IMOS platform code:IVER), capable of undertaking high resolution geo-referenced survey work. This platform is a modified Ocean Server Iver2 AUV that is hand deployable off RHIBs and other small vessels in addition to being deployable off larger vessels.\n\nThis AUV has been modified for benthic imaging, including the addition of USBL and DVL for more accurate navigation and the addition of high resolution stereo cameras and strobes. The submersible is equipped with a suite of oceanographic sensors including high resolution stereo cameras (6MP each), depth sensor, Doppler Velocity Log (DVL), Compass and Ultra Short Baseline (USBL). The vehicle is controlled by an on-board PC stack which is used to log sensor information and run the vehicle's control algorithms.\n\nThe vehicle has demonstrated its capacity to collect high resolution, near bottom imagery on trajectories over smooth terrain that has been used to generate 3D meshes and ortho-mosaics.\"}" + "title": "Any works" } ], "license": "Creative Commons Attribution 4.0 International License", @@ -11793,7 +11792,7 @@ "National Reef Monitoring Network Sub-Facility, Integrated Marine Observing System (IMOS)" ] }, - "sci:citation": "{\"suggestedCitation\":\"The citation in a list of references is: \\\"Reef Life Survey (RLS); Institute for Marine and Antarctic Studies (IMAS); Parks Victoria; Department of Primary Industries (DPI), New South Wales Government; Parks and Wildlife Tasmania; Department for Environment and Water (DEWNR), South Australia, Integrated Marine Observing System (IMOS) [year-of-data-download], National Reef Monitoring Network Sub-Facility, [data-access-URL], accessed [date-of-access].\\\"\",\"useLimitations\":[\"Data, products and services from IMOS are provided \\\"as is\\\" without any warranty as to fitness for a particular purpose.\"],\"otherConstraints\":null}", + "sci:citation": "Not importand for testing", "type": "Collection", "stac_version": "1.0.0", "stac_extensions": [ From 491d2a7059928252aedc28a47b926ab2defaa6f3 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 7 Oct 2025 18:23:59 +1100 Subject: [PATCH 05/24] missing profile in surefire --- server/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/pom.xml b/server/pom.xml index 3efac3b7..53ca5b50 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -266,6 +266,9 @@ maven-surefire-plugin --enable-preview + + test + From a8c1ad632c924d6d8edeac40ef9adbff2e7d4798 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 7 Oct 2025 22:43:49 +1100 Subject: [PATCH 06/24] missing profile in surefire --- .../au/org/aodn/ogcapi/server/features/RestApiTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 034b8425..06870636 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -290,14 +290,16 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(4, pageSize, "This test only works with small page"); + logger.info("Start verifyCorrectPageSizeAndScoreWithQuery"); + // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + "516811d7-cd1e-207a-e0440003ba8c79dd.json", // + "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", // * + "bc55eff4-7596-3565-e044-00144fdd4fa6.json", // "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result with search on "dataset" From a37d651c822dcb3b8cca59d6ef7a2aaf6002feca Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 08:34:16 +1100 Subject: [PATCH 07/24] clean before build --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1dee1f2..510aa52f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: cache: 'maven' - name: Build with Maven - run: mvn -B verify --file pom.xml + run: mvn clean -B verify --file pom.xml - name: Build Docker Image uses: docker/build-push-action@v5 From 9afc35db1197cda9e3cc90685895b726b969c05e Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 08:44:32 +1100 Subject: [PATCH 08/24] clean before build --- pom.xml | 2 +- .../test/java/au/org/aodn/ogcapi/server/BaseTestClass.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4515ae4..6bcf2f7b 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ co.elastic.clients elasticsearch-java - 8.13.3 + 8.13.4 org.mapstruct diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java index f76d0f59..45a6f0c7 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java @@ -7,6 +7,7 @@ import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; import co.elastic.clients.transport.rest_client.RestClientTransport; import com.fasterxml.jackson.databind.JsonNode; @@ -241,6 +242,9 @@ protected void insertJsonToElasticIndex(String index, String[] filenames) throws logger.debug(response.toString()); assertEquals(filenames.length, response.hits().hits().size(), "Number of docs stored is correct"); + for (Hit hit : response.hits().hits()) { + logger.info("Stored the following ids {}",hit.id()); + } } protected void insertJsonToElasticRecordIndex(String... filenames) throws IOException { From 966520c84a0e61296c3dc7158379c43234fa3aca Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 08:45:09 +1100 Subject: [PATCH 09/24] Add more log --- .../src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java | 2 +- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java index 45a6f0c7..0897e66f 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java @@ -243,7 +243,7 @@ protected void insertJsonToElasticIndex(String index, String[] filenames) throws assertEquals(filenames.length, response.hits().hits().size(), "Number of docs stored is correct"); for (Hit hit : response.hits().hits()) { - logger.info("Stored the following ids {}",hit.id()); + logger.debug("Stored the following ids {}",hit.id()); } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 06870636..29f8b1e9 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -290,7 +290,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(4, pageSize, "This test only works with small page"); - logger.info("Start verifyCorrectPageSizeAndScoreWithQuery"); + logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery"); // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. From b856ad4f18843b88c365f44c479f05991a0c99d3 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 09:08:10 +1100 Subject: [PATCH 10/24] Add more log --- .../test/java/au/org/aodn/ogcapi/server/BaseTestClass.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java index 0897e66f..3e90aa45 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java @@ -243,7 +243,9 @@ protected void insertJsonToElasticIndex(String index, String[] filenames) throws assertEquals(filenames.length, response.hits().hits().size(), "Number of docs stored is correct"); for (Hit hit : response.hits().hits()) { - logger.debug("Stored the following ids {}",hit.id()); + if(hit.source() != null) { + logger.debug("Stored the following id {}", hit.source().get("id")); + } } } From c889a8bc57806dd72a3f3035d9d10a8125f3501a Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 09:20:40 +1100 Subject: [PATCH 11/24] Fix wrong package name --- server/src/main/resources/log4j2-spring.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/log4j2-spring.xml b/server/src/main/resources/log4j2-spring.xml index 208e747d..6565e772 100644 --- a/server/src/main/resources/log4j2-spring.xml +++ b/server/src/main/resources/log4j2-spring.xml @@ -36,7 +36,7 @@ - + From 5e759df0d45d0696b56047620d51dee5496b2037 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 17:29:56 +1100 Subject: [PATCH 12/24] Add more log to debug --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 29f8b1e9..5be3b432 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -310,6 +310,8 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { new ParameterizedTypeReference<>() { }); + logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery - Done query 1"); + assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); // Given request page size is 1 assertEquals(1, @@ -345,6 +347,8 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { new ParameterizedTypeReference<>() { }); + logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery - Done query 2"); + assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); assertEquals(4, Objects.requireNonNull(collections.getBody()).getCollections().size(), From 51f19d94cad86e0c62381848bbede5531ac374eb Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 17:49:54 +1100 Subject: [PATCH 13/24] Add more log to debug --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 5be3b432..c391b770 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -310,7 +310,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { new ParameterizedTypeReference<>() { }); - logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery - Done query 1"); + logger.debug("verifyCorrectPageSizeAndScoreWithQuery - Done query 1"); assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); // Given request page size is 1 @@ -323,6 +323,8 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); + + logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( "95", collections.getBody().getSearchAfter().get(1), From 063a7ddb5cb8a775fb6be809a6f3687e502b3dda Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 18:05:16 +1100 Subject: [PATCH 14/24] Missing schema recreate --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index c391b770..f4a4d88d 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -44,6 +44,7 @@ public void clear() { @BeforeEach public void afterTest() { super.clearElasticIndex(); + super.createElasticIndex(); } @Test From 4002108251b4b4ede3dd47a51e4d29dd16b0a7e0 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 18:26:21 +1100 Subject: [PATCH 15/24] Print uuid for debug --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index f4a4d88d..4fb4720b 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -325,7 +325,9 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); + logger.debug("verifyCorrectPageSizeAndScoreWithQuery - uuid return {}", collections.getBody().getCollections().get(0).getId()); logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); + assertEquals( "95", collections.getBody().getSearchAfter().get(1), From cf6a49aec3a1cafbedbf8736e1d95ded52a337df Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 21:16:27 +1100 Subject: [PATCH 16/24] update schema --- .../core/model/enumeration/CQLFields.java | 3 +- .../portal_records_index_schema.json | 45 +++++++++---------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 3e68ab7d..ca7e2073 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -184,7 +184,8 @@ public enum CQLFields implements CQLFieldsInterface { CQLElasticSetting.score.getSetting(), CQLElasticSetting.score.getSetting(), null, - (order) -> new SortOptions.Builder().field(f -> f.field(CQLElasticSetting.score.getSetting()).order(order)) + (order) -> new SortOptions.Builder() + .field(f -> f.field(CQLElasticSetting.score.getSetting()).order(order)) ), // Rank score is an internal calculated score, it is different from the one use by ElasticSearch, // @see es-indexer RankingService diff --git a/server/src/test/resources/portal_records_index_schema.json b/server/src/test/resources/portal_records_index_schema.json index 62c4e28f..72b26ea9 100644 --- a/server/src/test/resources/portal_records_index_schema.json +++ b/server/src/test/resources/portal_records_index_schema.json @@ -141,28 +141,23 @@ "links": { "type": "nested", "properties": { - "link": { - "type": "nested", - "properties": { - "href": { - "type": "text" - }, - "rel": { - "type": "text" - }, - "type": { - "type": "text" - }, - "title": { - "type": "text" - }, - "description": { - "type": "text" - }, - "ai:group": { - "type": "text" - } - } + "href": { + "type": "text" + }, + "rel": { + "type": "text" + }, + "type": { + "type": "text" + }, + "title": { + "type": "text" + }, + "description": { + "type": "text" + }, + "ai:group": { + "type": "text" } } }, @@ -202,6 +197,9 @@ }, "summaries": { "properties": { + "ai:description": { + "type": "text" + }, "score": { "type": "long" }, @@ -253,9 +251,6 @@ }, "statement": { "type": "text" - }, - "ai:description": { - "type": "text" } } }, From 7bed4e70c17a160af898dc1f3329cfecfffaf81e Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 21:24:03 +1100 Subject: [PATCH 17/24] update schema --- .../ogcapi/server/features/RestApiTest.java | 10 ++--- .../portal_records_index_schema.json | 39 +++++++++++-------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 4fb4720b..68dda6c8 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -298,9 +298,9 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", // - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", // * - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", // + "516811d7-cd1e-207a-e0440003ba8c79dd.json", + "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + "bc55eff4-7596-3565-e044-00144fdd4fa6.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result with search on "dataset" @@ -329,12 +329,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( - "95", + "80", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); assertEquals( - "str:7709f541-fc0c-4318-b5b9-9053aa474e0e", + "str:bc55eff4-7596-3565-e044-00144fdd4fa6", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); diff --git a/server/src/test/resources/portal_records_index_schema.json b/server/src/test/resources/portal_records_index_schema.json index 72b26ea9..9d851676 100644 --- a/server/src/test/resources/portal_records_index_schema.json +++ b/server/src/test/resources/portal_records_index_schema.json @@ -141,23 +141,28 @@ "links": { "type": "nested", "properties": { - "href": { - "type": "text" - }, - "rel": { - "type": "text" - }, - "type": { - "type": "text" - }, - "title": { - "type": "text" - }, - "description": { - "type": "text" - }, - "ai:group": { - "type": "text" + "link": { + "type": "nested", + "properties": { + "href": { + "type": "text" + }, + "rel": { + "type": "text" + }, + "type": { + "type": "text" + }, + "title": { + "type": "text" + }, + "description": { + "type": "text" + }, + "ai:group": { + "type": "text" + } + } } } }, From c77746310b0f7cb95497b772420d7408124a6aea Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 21:35:14 +1100 Subject: [PATCH 18/24] rollback version change --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6bcf2f7b..e4515ae4 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ co.elastic.clients elasticsearch-java - 8.13.4 + 8.13.3 org.mapstruct From 86791a7cddb41b906d039c500d914b985cc69399 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 22:52:19 +1100 Subject: [PATCH 19/24] rollback version change --- .../ogcapi/server/features/RestApiTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 68dda6c8..d8a0db3f 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -587,4 +587,31 @@ public void verifyAggregationFeatureSummaryWithPageCorrect() throws IOException )); assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); } + + @Test + public void verifyCorrectPageSizeAndScoreWithQuery2() throws IOException { + assertEquals(4, pageSize, "This test only works with small page"); + + logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery"); + + // Given 6 records and we set page to 4, that means each query elastic return 4 record only + // and the logic to load the reset can kick in. + super.insertJsonToElasticRecordIndex( + "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + "19da2ce7-138f-4427-89de-a50c724f5f54.json", + "516811d7-cd1e-207a-e0440003ba8c79dd.json", + "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); + + // Call rest api directly and get query result with search on "dataset" + ResponseEntity collections = testRestTemplate.exchange( + getBasePath() + "/collections?q=dataset&filter=page_size=5 AND score>=1.3", + HttpMethod.GET, + null, + new ParameterizedTypeReference<>() {}); + + collections.getBody().getCollections().forEach(i -> logger.info("uuid {}", i.getId())); + + } } From 5221bed8dfbdeae40ee3b513efd8d8c1414fc192 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 22:57:01 +1100 Subject: [PATCH 20/24] Delete test change --- .../ogcapi/server/features/RestApiTest.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index d8a0db3f..68dda6c8 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -587,31 +587,4 @@ public void verifyAggregationFeatureSummaryWithPageCorrect() throws IOException )); assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); } - - @Test - public void verifyCorrectPageSizeAndScoreWithQuery2() throws IOException { - assertEquals(4, pageSize, "This test only works with small page"); - - logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery"); - - // Given 6 records and we set page to 4, that means each query elastic return 4 record only - // and the logic to load the reset can kick in. - super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", - "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); - - // Call rest api directly and get query result with search on "dataset" - ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=5 AND score>=1.3", - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() {}); - - collections.getBody().getCollections().forEach(i -> logger.info("uuid {}", i.getId())); - - } } From c172bb38ccdc87c480b9ef44c3e91929d6a0fc58 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 22:57:53 +1100 Subject: [PATCH 21/24] Add back --- .../ogcapi/server/features/RestApiTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 68dda6c8..d8a0db3f 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -587,4 +587,31 @@ public void verifyAggregationFeatureSummaryWithPageCorrect() throws IOException )); assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); } + + @Test + public void verifyCorrectPageSizeAndScoreWithQuery2() throws IOException { + assertEquals(4, pageSize, "This test only works with small page"); + + logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery"); + + // Given 6 records and we set page to 4, that means each query elastic return 4 record only + // and the logic to load the reset can kick in. + super.insertJsonToElasticRecordIndex( + "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + "19da2ce7-138f-4427-89de-a50c724f5f54.json", + "516811d7-cd1e-207a-e0440003ba8c79dd.json", + "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); + + // Call rest api directly and get query result with search on "dataset" + ResponseEntity collections = testRestTemplate.exchange( + getBasePath() + "/collections?q=dataset&filter=page_size=5 AND score>=1.3", + HttpMethod.GET, + null, + new ParameterizedTypeReference<>() {}); + + collections.getBody().getCollections().forEach(i -> logger.info("uuid {}", i.getId())); + + } } From 629e356c6b654582d315bb928a1c7b3d37f2f517 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 23:05:20 +1100 Subject: [PATCH 22/24] Add back --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index d8a0db3f..41cb7210 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -612,6 +612,6 @@ public void verifyCorrectPageSizeAndScoreWithQuery2() throws IOException { new ParameterizedTypeReference<>() {}); collections.getBody().getCollections().forEach(i -> logger.info("uuid {}", i.getId())); - + assertFalse(true); } } From 025f4eaad45a46c90c271fedaa7c5c6e9cf22bf5 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 23:15:11 +1100 Subject: [PATCH 23/24] Add back --- .../ogcapi/server/features/RestApiTest.java | 535 ------------------ 1 file changed, 535 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 41cb7210..b09761d4 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -52,541 +52,6 @@ public void afterTest() { public void verifyClusterIsHealthy() throws IOException { super.assertClusterHealthResponse(); } - /** - * We want to test the pageableSearch inside the elastic search is right or wrong by setting up more than 4 canned data, then - * query all to get them back even the search result return from elastic is break down into 4 + 2 - */ - @Test - public void verifyCorrectInternalPagingLargeData() throws IOException { - assertEquals(4, pageSize, "This test only works with small page"); - - // Given 6 records and we set page to 4, that means each query elastic return 4 record only - // and the logic to load the reset can kick in. - super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", - "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); - - // Call rest api directly and get query result - ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections", - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() {}); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - assertEquals(6, Objects.requireNonNull(collections.getBody()).getCollections().size(), "Total equals"); - assertEquals(6, collections.getBody().getTotal(), "Get total works"); - - // Now make sure all id exist - Set ids = new HashSet<>(List.of( - "5c418118-2581-4936-b6fd-d6bedfe74f62", - "19da2ce7-138f-4427-89de-a50c724f5f54", - "516811d7-cd1e-207a-e0440003ba8c79dd", - "7709f541-fc0c-4318-b5b9-9053aa474e0e", - "bc55eff4-7596-3565-e044-00144fdd4fa6", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011" - )); - - for(Collection collection : Objects.requireNonNull(collections.getBody()).getCollections()) { - assertTrue(ids.contains(collection.getId()),"Contains " + collection.getId()); - } - } - /** - * with page_size set, the max number of record return will equals page_size - */ - @Test - public void verifyCorrectPageSizeDataReturn() throws IOException { - assertEquals(4, pageSize, "This test only works with small page"); - - // Given 6 records and we set page to 4, that means each query elastic return 4 record only - // and the logic to load the reset can kick in. - super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", - "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); - - // Call rest api directly and get query result - ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?filter=page_size=3", - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() {}); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - // Given request page size is 3, only 3 return this time - assertEquals(3, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct" - ); - // Total number of record should be this - assertEquals(6, collections.getBody().getTotal(), "Get total works"); - - // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); - assertEquals("1.0", collections.getBody().getSearchAfter().get(0), "Search after 1 value"); - assertEquals( - "100", - collections.getBody().getSearchAfter().get(1), - "search_after 2 arg" - ); - assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", - collections.getBody().getSearchAfter().get(2), - "search_after 3 arg" - ); - - // Now make sure all id exist - Set ids = new HashSet<>(List.of( - "5c418118-2581-4936-b6fd-d6bedfe74f62", - "19da2ce7-138f-4427-89de-a50c724f5f54", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011" - )); - - for(Collection collection : Objects.requireNonNull(collections.getBody()).getCollections()) { - assertTrue(ids.contains(collection.getId()),"Contains " + collection.getId()); - } - - // Now if we provided the search after we should get the next batch - collections = testRestTemplate.exchange( - getBasePath() + "/collections?filter=page_size=3 AND search_after=" + - String.format("'%s||%s||%s'", - collections.getBody().getSearchAfter().get(0), - collections.getBody().getSearchAfter().get(1), - collections.getBody().getSearchAfter().get(2)), - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() {}); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - // Given request page size is 3, only 3 return this time - assertEquals(3, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct" - ); - - ids = new HashSet<>(List.of( - "7709f541-fc0c-4318-b5b9-9053aa474e0e", - "bc55eff4-7596-3565-e044-00144fdd4fa6", - "516811d7-cd1e-207a-e0440003ba8c79dd" - )); - - for(Collection collection : Objects.requireNonNull(collections.getBody()).getCollections()) { - assertTrue(ids.contains(collection.getId()),"Contains in next batch " + collection.getId()); - } - } - /** - * Extreme case, page size set to 1 and query text "dataset" and page one by one. Only part of the json - * will be return, the sort value should give you the next item and you will be able to go to next one. - * The first sort value is the relevant and because of query text the value will be something greater than 1.0 - */ - @Test - public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { - assertEquals(4, pageSize, "This test only works with small page"); - - // Given 6 records and we set page to 4, that means each query elastic return 4 record only - // and the logic to load the reset can kick in. - super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", - "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); - - // Call rest api directly and get query result with search on "dataset" - ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=1", - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() { - }); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - // Given request page size is 1 - assertEquals(1, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct" - ); - // Total number of record should be this - assertEquals(5, collections.getBody().getTotal(), "Get total works"); - - // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); - assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", - collections.getBody().getSearchAfter().get(2), - "search_after 2 arg" - ); - - // Now the same search, same page but search_after the result above given sort value - // intended to give space after comma for negative test - collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=1 AND search_after=" + - String.format("'%s||%s||%s'", - collections.getBody().getSearchAfter().get(0), - collections.getBody().getSearchAfter().get(1), - "bc55eff4-7596-3565-e044-00144fdd4fa6"), - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() { - }); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - assertEquals(1, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct" - ); - // Total number of record should be this as the same search criteria applies - assertEquals(5, collections.getBody().getTotal(), "Get total works"); - - // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); - assertEquals( - "str:7709f541-fc0c-4318-b5b9-9053aa474e0e", - collections.getBody().getSearchAfter().get(2), - "search_after 3 arg" - ); - - // Now the same search, diff page but search_after the result above given sort value - // set a bigger page size which exceed more than record hit as negative test - collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=3 AND search_after=" + - String.format("'%s||%s ||%s'", - collections.getBody().getSearchAfter().get(0), - collections.getBody().getSearchAfter().get(1), - "5c418118-2581-4936-b6fd-d6bedfe74f62"), - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() { - }); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - assertEquals(3, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct, total hit is 4, we move to the third record" - ); - // Total number of record should be this as the same search criteria applies - assertEquals(5, collections.getBody().getTotal(), "Get total works"); - - // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); - assertEquals( - "str:19da2ce7-138f-4427-89de-a50c724f5f54", - collections.getBody().getSearchAfter().get(2), - "search_after 3 value" - ); - } - /** - * Similar to verifyCorrectPageSizeDataReturnWithQuery and add score in the query, - * this is used to verify a bug fix where page_size and score crash the query - */ - @Test - public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { - assertEquals(4, pageSize, "This test only works with small page"); - - logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery"); - - // Given 6 records and we set page to 4, that means each query elastic return 4 record only - // and the logic to load the reset can kick in. - super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", - "19da2ce7-138f-4427-89de-a50c724f5f54.json", - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - "bc55eff4-7596-3565-e044-00144fdd4fa6.json", - "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); - - // Call rest api directly and get query result with search on "dataset" - ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=1.3", - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() { - }); - - logger.debug("verifyCorrectPageSizeAndScoreWithQuery - Done query 1"); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - // Given request page size is 1 - assertEquals(1, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct" - ); - // Total number of record should be this - assertEquals(5, collections.getBody().getTotal(), "Get total works"); - - // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); - - logger.debug("verifyCorrectPageSizeAndScoreWithQuery - uuid return {}", collections.getBody().getCollections().get(0).getId()); - logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); - - assertEquals( - "80", - collections.getBody().getSearchAfter().get(1), - "search_after 2 value" - ); - assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", - collections.getBody().getSearchAfter().get(2), - "search_after 3 value" - ); - - // Now the same search, same page but search_after the result above given sort value - // intended to give space after comma for negative test - collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=1.3 AND search_after=" + - String.format("'%s|| %s || %s'", - collections.getBody().getSearchAfter().get(0), - collections.getBody().getSearchAfter().get(1), - "bc55eff4-7596-3565-e044-00144fdd4fa6"), - HttpMethod.GET, - null, - new ParameterizedTypeReference<>() { - }); - - logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery - Done query 2"); - - assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); - assertEquals(4, - Objects.requireNonNull(collections.getBody()).getCollections().size(), - "Record return size correct" - ); - // Total number of record should be this as the same search criteria applies - assertEquals(5, collections.getBody().getTotal(), "Get total works"); - - // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); - assertEquals( - "str:5c418118-2581-4936-b6fd-d6bedfe74f62", - collections.getBody().getSearchAfter().get(2), - "Search after 2 value" - ); - } - - @Test - public void verifyGetSingleCollection() throws IOException { - super.insertJsonToElasticRecordIndex( - "516811d7-cd1e-207a-e0440003ba8c79dd.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" - ); - - // Call rest api directly and get query result - ResponseEntity collection = testRestTemplate.getForEntity( - getBasePath() + "/collections/516811d7-cd1e-207a-e0440003ba8c79dd", - Collection.class); - - assertNotNull(collection.getBody(), "Body not null"); - assertEquals( - "516811d7-cd1e-207a-e0440003ba8c79dd", - collection.getBody().getId(), - "Correct UUID - 516811d7-cd1e-207a-e0440003ba8c79dd"); - } - - @Test - public void verifyBBoxCorrect() throws IOException { - super.insertJsonToElasticRecordIndex( - "ae86e2f5-eaaf-459e-a405-e654d85adb9c.json", - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" - ); - - // Call rest api directly and get query result - ResponseEntity collection = testRestTemplate.getForEntity( - getBasePath() + "/collections/ae86e2f5-eaaf-459e-a405-e654d85adb9c", - Collection.class); - - assertNotNull(collection.getBody(), "Body not null"); - - List> bbox = collection.getBody().getExtent().getSpatial().getBbox(); - assertEquals( - 24, - bbox.size(), - "Count of bbox"); - - // Should be something like this but order may be diff - // "bbox" : [ - // [ 113.0, -43.0, 154.0, -9.0 ], [ 115.0, -21.0, 117.0, -19.0 ], [ 114.0, -21.0, 115.0, -20.0 ], - // [ 152.0, -22.0, 153.0, -21.0 ], [ 113.0, -22.0, 114.0, -21.0 ], [ 151.0, -24.0, 153.0, -22.0 ], - // [ 130.0, -10.0, 131.0, -9.0 ], [ 121.0, -17.0, 122.0, -15.0 ], [ 130.0, -13.0, 131.0, -12.0 ], - // [ 127.0, -14.0, 129.0, -9.0 ], [ 145.0, -15.0, 146.0, -14.0 ], [ 123.0, -15.0, 124.0, -14.0 ], - // [ 119.0, -18.0, 120.0, -17.0 ], [ 147.0, -20.0, 148.0, -18.0 ], [ 153.0, -28.0, 154.0, -27.0 ], - // [ 153.0, -31.0, 154.0, -30.0 ], [ 137.0, -34.0, 138.0, -33.0 ], [ 114.0, -33.0, 116.0, -31.0 ], - // [ 121.0, -34.0, 122.0, -33.0 ], [ 151.0, -35.0, 152.0, -33.0 ], [ 150.0, -37.0, 151.0, -36.0 ], - // [ 134.0, -37.0, 137.0, -34.0 ], [ 141.0, -39.0, 142.0, -38.0 ], [ 148.0, -43.0, 149.0, -42.0 ] ], - Optional> target = bbox.stream() - .filter(box -> box.get(0).doubleValue() == 141.0) - .filter(box -> box.get(1).doubleValue() == -39.0) - .filter(box -> box.get(2).doubleValue() == 142.0) - .filter(box -> box.get(3).doubleValue() == -38.0) - .findFirst(); - - assertTrue(target.isPresent(), "Target bbox found 1"); - - target = bbox.stream() - .filter(box -> box.get(0).doubleValue() == 152.0) - .filter(box -> box.get(1).doubleValue() == -22.0) - .filter(box -> box.get(2).doubleValue() == 153.0) - .filter(box -> box.get(3).doubleValue() == -21.0) - .findFirst(); - - assertTrue(target.isPresent(), "Target bbox found 2"); - - logger.info(bbox.get(0).toString()); - // The first is the overall bounding box - assertEquals(113.0, bbox.get(0).get(0).doubleValue(), "Overall bounding box coor 1"); - assertEquals(-43.0, bbox.get(0).get(1).doubleValue(), "Overall bounding box coor 2"); - assertEquals(154.0, bbox.get(0).get(2).doubleValue(), "Overall bounding box coor 3"); - assertEquals(-9.0, bbox.get(0).get(3).doubleValue(), "Overall bounding box coor 4"); - } - /** - * Verify the function correctly sum up the values for feature id summary - * @throws IOException - Not expect to throw - */ - @Disabled("Skipping this test temporarily") - @Test - public void verifyAggregationFeatureSummaryCorrect() throws IOException { - super.insertJsonToElasticCODataIndex( - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json" - ); - - // Call rest api directly and get query result - ResponseEntity collection = testRestTemplate.getForEntity( - getBasePath() + "/collections/35234913-aa3c-48ec-b9a4-77f822f66ef8/items/summary", - FeatureCollectionGeoJSON.class); - - assertNotNull(collection.getBody(), "Body not null"); - - FeatureCollectionGeoJSON json = collection.getBody(); - assertEquals(3, json.getFeatures().size(), "Features correct"); - - // Sort make sure compare always same order - List sf = json.getFeatures().stream() - .sorted((a,b) -> b.getGeometry().hashCode() - a.getGeometry().hashCode()) - .toList(); - // Sample1 - FeatureGeoJSON featureGeoJSON1 = new FeatureGeoJSON(); - featureGeoJSON1.setType(FeatureGeoJSON.TypeEnum.FEATURE); - featureGeoJSON1.setGeometry(new PointGeoJSON() - .type(PointGeoJSON.TypeEnum.POINT) - .coordinates(List.of(BigDecimal.valueOf(159.26), BigDecimal.valueOf(-24.72))) - ); - featureGeoJSON1.setProperties(Map.of( - FeatureProperty.COUNT.getValue(), 42.0, - FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", - FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" - - )); - assertEquals(featureGeoJSON1, sf.get(0), "featureGeoJSON1"); - - // Sample3 - FeatureGeoJSON featureGeoJSON2 = new FeatureGeoJSON(); - featureGeoJSON2.setType(FeatureGeoJSON.TypeEnum.FEATURE); - featureGeoJSON2.setGeometry(new PointGeoJSON() - .type(PointGeoJSON.TypeEnum.POINT) - .coordinates(List.of(BigDecimal.valueOf(154.81), BigDecimal.valueOf(-26.2))) - ); - featureGeoJSON2.setProperties(Map.of( - FeatureProperty.COUNT.getValue(), 48.0, - FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", - FeatureProperty.END_TIME.getValue(), "2024-03-01T00:00:00.000Z" - - )); - assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); - - FeatureGeoJSON featureGeoJSON3 = new FeatureGeoJSON(); - featureGeoJSON3.setType(FeatureGeoJSON.TypeEnum.FEATURE); - featureGeoJSON3.setGeometry(new PointGeoJSON() - .type(PointGeoJSON.TypeEnum.POINT) - .coordinates(List.of(BigDecimal.valueOf(153.56), BigDecimal.valueOf(-26.59))) - ); - featureGeoJSON3.setProperties(Map.of( - FeatureProperty.COUNT.getValue(), 14.0, - FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", - FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" - - )); - assertEquals(featureGeoJSON3, sf.get(2), "featureGeoJSON3"); - } - /** - * We add more sample data and will trigger page load. - * @throws IOException - Not expect to throw - */ - @Disabled("Skipping this test temporarily") - @Test - public void verifyAggregationFeatureSummaryWithPageCorrect() throws IOException { - assertEquals(4, pageSize, "This test only works with small page"); - - super.insertJsonToElasticCODataIndex( - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json", - "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json" - ); - - // Call rest api directly and get query result - ResponseEntity collection = testRestTemplate.getForEntity( - getBasePath() + "/collections/35234913-aa3c-48ec-b9a4-77f822f66ef8/items/summary", - FeatureCollectionGeoJSON.class); - - assertNotNull(collection.getBody(), "Body not null"); - - FeatureCollectionGeoJSON json = collection.getBody(); - assertEquals(5, json.getFeatures().size(), "Features correct"); - - // Sort make sure compare always same order - List sf = json.getFeatures().stream() - .sorted((a,b) -> b.getGeometry().hashCode() - a.getGeometry().hashCode()) - .toList(); - - // Sample1 - FeatureGeoJSON featureGeoJSON1 = new FeatureGeoJSON(); - featureGeoJSON1.setType(FeatureGeoJSON.TypeEnum.FEATURE); - featureGeoJSON1.setGeometry(new PointGeoJSON() - .type(PointGeoJSON.TypeEnum.POINT) - .coordinates(List.of(BigDecimal.valueOf(163.56), BigDecimal.valueOf(-26.59))) - ); - featureGeoJSON1.setProperties(Map.of( - FeatureProperty.COUNT.getValue(), 14.0, - FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", - FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" - - )); - assertEquals(featureGeoJSON1, sf.get(0), "featureGeoJSON1"); - - // Sample5 - FeatureGeoJSON featureGeoJSON2 = new FeatureGeoJSON(); - featureGeoJSON2.setType(FeatureGeoJSON.TypeEnum.FEATURE); - featureGeoJSON2.setGeometry(new PointGeoJSON() - .type(PointGeoJSON.TypeEnum.POINT) - .coordinates(List.of(BigDecimal.valueOf(163.56), BigDecimal.valueOf(-126.59))) - ); - featureGeoJSON2.setProperties(Map.of( - FeatureProperty.COUNT.getValue(), 20.0, - FeatureProperty.START_TIME.getValue(), "2022-12-01T00:00:00.000Z", - FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" - - )); - assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); - } @Test public void verifyCorrectPageSizeAndScoreWithQuery2() throws IOException { From c07bb1a491e95fd74345e11a40c70a3e083d25c9 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 8 Oct 2025 23:33:37 +1100 Subject: [PATCH 24/24] Add back --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index b09761d4..9a8b6fa4 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -71,12 +71,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery2() throws IOException { // Call rest api directly and get query result with search on "dataset" ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=5 AND score>=1.3", + getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=1.3", HttpMethod.GET, null, new ParameterizedTypeReference<>() {}); - collections.getBody().getCollections().forEach(i -> logger.info("uuid {}", i.getId())); + collections.getBody().getCollections().forEach(i -> logger.info("uuid {}, {}", i.getId(), collections.getBody().getSearchAfter())); assertFalse(true); } }