From 6bc895a773abe4b2122ae1283f3b9d903451d0ef Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 7 Nov 2025 14:52:52 +1100 Subject: [PATCH 01/10] use script score for elasticsearch query --- .../core/service/CacheNoLandGeometry.java | 3 +- .../server/core/service/ElasticSearch.java | 19 ++- .../core/service/ElasticSearchBase.java | 114 ++++++++++++++++-- 3 files changed, 121 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java index c032c0ac..befb5d94 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java @@ -38,7 +38,8 @@ public Map getAllNoLandGeometry() { null, null, null, - null); + null, + false); return result.collections .stream() diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 15779168..ff76b4ac 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -211,7 +211,8 @@ protected ElasticSearchBase.SearchResult searchCollectionsB null, createSortOptions(sortBy, CQLFields.class), null, - null); + null, + false); } @Override @@ -240,6 +241,11 @@ public ElasticSearchBase.SearchResult searchAllCollections( return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } + /*** + * This function is used for searching by user input query. In such case, the sorting is from two impactors: the relavance score (powered by Elasticsearch) in the `._score` field, + * and the importance score (powered by IMOS internal ranking, which has a maximum value of 106). + * The final score is relavance_score * (importance_score / max(importance_score)) + */ @Override public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { @@ -333,6 +339,14 @@ public ElasticSearchBase.SearchResult searchByParameters(Li .toList(); } + boolean useScriptScore = false; + if (sortBy != null && !sortBy.isEmpty()) { + // only use script_score if sortby contains "score" and should field is not empty + if (sortBy.toLowerCase().contains("score") && should != null && !should.isEmpty()) { + useScriptScore = true; + } + } + return searchCollectionBy( null, should, @@ -341,7 +355,8 @@ public ElasticSearchBase.SearchResult searchByParameters(Li searchAfter, createSortOptions(sortBy, CQLFields.class), score, - maxSize + maxSize, + useScriptScore ); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 75059eeb..6e1dba13 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -156,8 +156,8 @@ protected SearchResult searchCollectionBy(final List final List searchAfter, final List sortOptions, final Double score, - final Long maxSize) { - + final Long maxSize, + final boolean useScriptScore) { Supplier builderSupplier = () -> { SearchRequest.Builder builder = new SearchRequest.Builder(); builder.index(indexName) @@ -165,23 +165,113 @@ protected SearchResult searchCollectionBy(final List // we use the smaller one. The internal page size is used to get the result by // batch, lets say page is 20 and internal is 10, then we do it in two batch. // But if we request 5 only, then there is no point to load 10 - .size(maxSize != null && maxSize < pageSize ? maxSize.intValue() : pageSize) - .query(q -> q.bool(createBoolQueryForProperties(queries, should, filters))); + .size(maxSize != null && maxSize < pageSize ? maxSize.intValue() : pageSize); + + // use script score if search with text, in such case, the final score depends on both relevance and metadata quality + // put query in script block + if (useScriptScore) { + builder.query(q -> q.scriptScore(ss -> ss + // to get the original _score from ELasticsearch + .query(bq -> bq.bool(createBoolQueryForProperties(queries, should, filters))) + .script(s -> s.inline(i -> i + .lang("painless") + .source( + // Step 1: Retrieve internal quality score from summaries.score field + // Default to 0 if field doesn't exist or is empty + "double internalScore = doc.containsKey('summaries.score') && " + + "!doc['summaries.score'].empty ? doc['summaries.score'].value : 0.0; " + + + // Step 2: Normalize internal score to 0-1 range + // Assuming summaries.score is in range 0-106 + "double normalizedScore = internalScore / 106.0; " + + + // Step 3: Ensure minimum multiplier to avoid zero scores + "double multiplier = Math.max(normalizedScore, 0.01); " + + + // Step 4: Calculate final score + // Final score = Elasticsearch relevance * normalized quality + "return _score * multiplier;" + ) + ) + )) + ); + } + // use original query logic + else { + builder.query(q -> q.bool(createBoolQueryForProperties(queries, should, filters))); + } if(searchAfter != null) { builder.searchAfter(searchAfter); } - if(sortOptions != null) { - builder.sort(sortOptions); + // to use sort by uuid as a tiebreaker + boolean hasUuidSort = false; + + // apply sort options + if (useScriptScore) { + // when using script_score, always sort by _score first + builder.sort(so -> so.score(sc -> sc.order(SortOrder.Desc))); + + // add other sort options + if (sortOptions != null && !sortOptions.isEmpty()) { + for (SortOptions sortOption : sortOptions) { + // skip score sort if no relevance score applied (equally as 1 or null) + if (sortOption.isScore() && (should == null || should.isEmpty())) { + continue; + } + builder.sort(sortOption); + + // check if it has sort by id option + if (sortOption.isField() && + sortOption.field().field().equals(StacBasicField.UUID.sortField)) { + hasUuidSort = true; + } + } + } } + else { + // when not using script_score, apply all sort options + if (sortOptions != null && !sortOptions.isEmpty()) { + // check if sortOptions already contains _score + boolean hasScoreSort = sortOptions.stream() + .anyMatch(SortOptions::isScore); + + // if there are text queries (should clause) but no _score in sortOptions, + // add _score as first sort criterion + if (!hasScoreSort && should != null && !should.isEmpty()) { + builder.sort(so -> so.score(sc -> sc.order(SortOrder.Desc))); + } - builder.sort(so -> so - // We need a unique key for the search, cannot use _id in v8 anymore, so we need - // to sort using the keyword, this field is not for search and therefore not in enum - .field(FieldSort.of(f -> f - .field(StacBasicField.UUID.sortField) - .order(SortOrder.Asc)))); + for (SortOptions sortOption : sortOptions) { + // skip score sort if no relevance score applied (equally as 1 or null) + if (sortOption.isScore() && (should == null || should.isEmpty())) { + continue; + } + builder.sort(sortOption); + + // check if it has sort by id option + if (sortOption.isField() && + sortOption.field().field().equals(StacBasicField.UUID.sortField)) { + hasUuidSort = true; + } + } + } + else if (should != null && !should.isEmpty()) { + // If no sortOptions provided but there are text queries, + // default to sorting by _score + builder.sort(so -> so.score(sc -> sc.order(SortOrder.Desc))); + } + } + // add sort by id as the final tiebreaker if it was applied + if (!hasUuidSort) { + builder.sort(so -> so + // We need a unique key for the search, cannot use _id in v8 anymore, so we need + // to sort using the keyword, this field is not for search and therefore not in enum + .field(FieldSort.of(f -> f + .field(StacBasicField.UUID.sortField) + .order(SortOrder.Asc)))); + } if(score != null) { // By default we do not setup any min_score, the api caller should pass it in so From 140cd9a7741d3f947e54bdfc7e23385d2489c607 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 7 Nov 2025 15:59:45 +1100 Subject: [PATCH 02/10] fix tests --- .../core/service/ElasticSearchBase.java | 23 +------------------ .../ogcapi/server/features/RestApiTest.java | 12 +++++----- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 6e1dba13..eb495eb9 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -210,16 +210,9 @@ protected SearchResult searchCollectionBy(final List // apply sort options if (useScriptScore) { - // when using script_score, always sort by _score first - builder.sort(so -> so.score(sc -> sc.order(SortOrder.Desc))); - - // add other sort options + // add sort options if (sortOptions != null && !sortOptions.isEmpty()) { for (SortOptions sortOption : sortOptions) { - // skip score sort if no relevance score applied (equally as 1 or null) - if (sortOption.isScore() && (should == null || should.isEmpty())) { - continue; - } builder.sort(sortOption); // check if it has sort by id option @@ -233,21 +226,7 @@ protected SearchResult searchCollectionBy(final List else { // when not using script_score, apply all sort options if (sortOptions != null && !sortOptions.isEmpty()) { - // check if sortOptions already contains _score - boolean hasScoreSort = sortOptions.stream() - .anyMatch(SortOptions::isScore); - - // if there are text queries (should clause) but no _score in sortOptions, - // add _score as first sort criterion - if (!hasScoreSort && should != null && !should.isEmpty()) { - builder.sort(so -> so.score(sc -> sc.order(SortOrder.Desc))); - } - for (SortOptions sortOption : sortOptions) { - // skip score sort if no relevance score applied (equally as 1 or null) - if (sortOption.isScore() && (should == null || should.isEmpty())) { - continue; - } builder.sort(sortOption); // check if it has sort by id option diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 68dda6c8..67b206fa 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -220,7 +220,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 2 arg" ); @@ -249,7 +249,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:7709f541-fc0c-4318-b5b9-9053aa474e0e", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 3 arg" ); @@ -278,7 +278,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:19da2ce7-138f-4427-89de-a50c724f5f54", + "str:bc55eff4-7596-3565-e044-00144fdd4fa6", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -329,12 +329,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( - "80", + "100", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -365,7 +365,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:5c418118-2581-4936-b6fd-d6bedfe74f62", + "str:19da2ce7-138f-4427-89de-a50c724f5f54", collections.getBody().getSearchAfter().get(2), "Search after 2 value" ); From cb7716845c8f2b2e056f72fd651750ebfdc9d9dd Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 7 Nov 2025 16:17:58 +1100 Subject: [PATCH 03/10] fix failed tests --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 67b206fa..4a728ef2 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -220,7 +220,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", + "str:bc55eff4-7596-3565-e044-00144fdd4fa6", collections.getBody().getSearchAfter().get(2), "search_after 2 arg" ); @@ -329,7 +329,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( - "100", + "80", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); From f82f8d882c7fbcd607e5276f09a848ee5c3a2304 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Mon, 10 Nov 2025 13:52:27 +1100 Subject: [PATCH 04/10] fix failed tests --- .../au/org/aodn/ogcapi/server/features/RestApiTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 4a728ef2..68dda6c8 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -249,7 +249,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", + "str:7709f541-fc0c-4318-b5b9-9053aa474e0e", collections.getBody().getSearchAfter().get(2), "search_after 3 arg" ); @@ -278,7 +278,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:19da2ce7-138f-4427-89de-a50c724f5f54", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -334,7 +334,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { "search_after 2 value" ); assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", + "str:bc55eff4-7596-3565-e044-00144fdd4fa6", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -365,7 +365,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:19da2ce7-138f-4427-89de-a50c724f5f54", + "str:5c418118-2581-4936-b6fd-d6bedfe74f62", collections.getBody().getSearchAfter().get(2), "Search after 2 value" ); From 2ea33739735d4f74dd8b38afa84a6b423e9a547c Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 11 Nov 2025 09:47:34 +1100 Subject: [PATCH 05/10] fix failed tests --- .../ogcapi/server/features/RestApiTest.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 68dda6c8..bfc333dc 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -185,6 +185,11 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { * Extreme case, page size set to 1 and query text "dataset" and page one by one. Only part of the json * will be return, the sort value should give you the next item and you will be able to go to next one. * The first sort value is the relevant and because of query text the value will be something greater than 1.0 + * After weighted sorting, the actual order is (for the first 4 records): + * Document 0: UUID=bf287dfe-9ce4-4969-9c59-51c39ea4d011 + * Document 1: UUID=7709f541-fc0c-4318-b5b9-9053aa474e0e + * Document 2: UUID=19da2ce7-138f-4427-89de-a50c724f5f54 + * Document 3: UUID=bc55eff4-7596-3565-e044-00144fdd4fa6 */ @Test public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { @@ -220,7 +225,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 2 arg" ); @@ -232,7 +237,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { String.format("'%s||%s||%s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), - "bc55eff4-7596-3565-e044-00144fdd4fa6"), + "bf287dfe-9ce4-4969-9c59-51c39ea4d011"), HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -286,6 +291,11 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { /** * Similar to verifyCorrectPageSizeDataReturnWithQuery and add score in the query, * this is used to verify a bug fix where page_size and score crash the query + * After weighted sorting, the actual order is (for the first 4 records): + * Document 0: UUID=bf287dfe-9ce4-4969-9c59-51c39ea4d011 + * Document 1: UUID=7709f541-fc0c-4318-b5b9-9053aa474e0e + * Document 2: UUID=19da2ce7-138f-4427-89de-a50c724f5f54 + * Document 3: UUID=bc55eff4-7596-3565-e044-00144fdd4fa6 */ @Test public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { @@ -329,12 +339,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( - "80", + "100", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -365,7 +375,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:5c418118-2581-4936-b6fd-d6bedfe74f62", + "str:19da2ce7-138f-4427-89de-a50c724f5f54", collections.getBody().getSearchAfter().get(2), "Search after 2 value" ); From 7d43f091b9679b08e5901efa540e9c903eec9289 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 11 Nov 2025 09:54:53 +1100 Subject: [PATCH 06/10] fix failed tests --- .../au/org/aodn/ogcapi/server/features/RestApiTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index bfc333dc..91f5c8d2 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -225,7 +225,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", + "str:bc55eff4-7596-3565-e044-00144fdd4fa6", collections.getBody().getSearchAfter().get(2), "search_after 2 arg" ); @@ -339,12 +339,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( - "100", + "80", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", + "str:bc55eff4-7596-3565-e044-00144fdd4fa6", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); From fd7016ff0af00cf096279e9bd7bedc1d824a2e96 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 11 Nov 2025 11:00:35 +1100 Subject: [PATCH 07/10] update test samples and fix failed tests --- .../ogcapi/server/common/RestApiTest.java | 2 +- .../ogcapi/server/features/RestApiTest.java | 55 ++++++++++++++----- .../19da2ce7-138f-4427-89de-a50c724f5f54.json | 2 +- .../5c418118-2581-4936-b6fd-d6bedfe74f62.json | 2 +- .../7709f541-fc0c-4318-b5b9-9053aa474e0e.json | 2 +- 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java index d6fae3fe..08ee28d4 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java @@ -526,7 +526,7 @@ public void verifyCQLPropertyScore() throws IOException { // Increase score will drop one record collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q='dataset includes'&filter=score>=3", Collections.class); - assertEquals(3, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 3, with score 3"); + assertEquals(2, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 3, with score 3"); assertEquals("bf287dfe-9ce4-4969-9c59-51c39ea4d011", Objects.requireNonNull(collections.getBody()).getCollections().get(0).getId(), "bf287dfe-9ce4-4969-9c59-51c39ea4d011"); } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 91f5c8d2..75c9751e 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -63,11 +63,17 @@ public void verifyCorrectInternalPagingLargeData() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. super.insertJsonToElasticRecordIndex( + // set test summaries.score 90 "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + // set test summaries.score 106 "19da2ce7-138f-4427-89de-a50c724f5f54.json", + // set test summaries.score 70 "516811d7-cd1e-207a-e0440003ba8c79dd.json", + // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + // set test summaries.score 80 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result @@ -97,6 +103,7 @@ public void verifyCorrectInternalPagingLargeData() throws IOException { } /** * with page_size set, the max number of record return will equals page_size + * With default search, the sort should follow uuid order */ @Test public void verifyCorrectPageSizeDataReturn() throws IOException { @@ -105,11 +112,17 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. super.insertJsonToElasticRecordIndex( + // set test summaries.score 90 "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + // set test summaries.score 106 "19da2ce7-138f-4427-89de-a50c724f5f54.json", + // set test summaries.score 70 "516811d7-cd1e-207a-e0440003ba8c79dd.json", + // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + // set test summaries.score 80 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result @@ -132,12 +145,12 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals("1.0", collections.getBody().getSearchAfter().get(0), "Search after 1 value"); assertEquals( - "100", + "90", collections.getBody().getSearchAfter().get(1), "search_after 2 arg" ); assertEquals( - "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", + "str:5c418118-2581-4936-b6fd-d6bedfe74f62", collections.getBody().getSearchAfter().get(2), "search_after 3 arg" ); @@ -187,9 +200,9 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { * The first sort value is the relevant and because of query text the value will be something greater than 1.0 * After weighted sorting, the actual order is (for the first 4 records): * Document 0: UUID=bf287dfe-9ce4-4969-9c59-51c39ea4d011 - * Document 1: UUID=7709f541-fc0c-4318-b5b9-9053aa474e0e - * Document 2: UUID=19da2ce7-138f-4427-89de-a50c724f5f54 - * Document 3: UUID=bc55eff4-7596-3565-e044-00144fdd4fa6 + * Document 1: UUID=19da2ce7-138f-4427-89de-a50c724f5f54 + * Document 2: UUID=bc55eff4-7596-3565-e044-00144fdd4fa6 + * Document 3: UUID=7709f541-fc0c-4318-b5b9-9053aa474e0e */ @Test public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { @@ -198,11 +211,17 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. super.insertJsonToElasticRecordIndex( + // set test summaries.score 90 "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + // set test summaries.score 106 "19da2ce7-138f-4427-89de-a50c724f5f54.json", + // set test summaries.score 70 "516811d7-cd1e-207a-e0440003ba8c79dd.json", + // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + // set test summaries.score 80 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result with search on "dataset" @@ -225,7 +244,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 2 arg" ); @@ -254,7 +273,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:7709f541-fc0c-4318-b5b9-9053aa474e0e", + "str:19da2ce7-138f-4427-89de-a50c724f5f54", collections.getBody().getSearchAfter().get(2), "search_after 3 arg" ); @@ -283,7 +302,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:19da2ce7-138f-4427-89de-a50c724f5f54", + "str:5c418118-2581-4936-b6fd-d6bedfe74f62", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -293,9 +312,9 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { * this is used to verify a bug fix where page_size and score crash the query * After weighted sorting, the actual order is (for the first 4 records): * Document 0: UUID=bf287dfe-9ce4-4969-9c59-51c39ea4d011 - * Document 1: UUID=7709f541-fc0c-4318-b5b9-9053aa474e0e - * Document 2: UUID=19da2ce7-138f-4427-89de-a50c724f5f54 - * Document 3: UUID=bc55eff4-7596-3565-e044-00144fdd4fa6 + * Document 1: UUID=19da2ce7-138f-4427-89de-a50c724f5f54 + * Document 2: UUID=bc55eff4-7596-3565-e044-00144fdd4fa6 + * Document 3: UUID=7709f541-fc0c-4318-b5b9-9053aa474e0e */ @Test public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { @@ -306,11 +325,17 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. super.insertJsonToElasticRecordIndex( + // set test summaries.score 90 "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + // set test summaries.score 106 "19da2ce7-138f-4427-89de-a50c724f5f54.json", + // set test summaries.score 70 "516811d7-cd1e-207a-e0440003ba8c79dd.json", + // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", + // set test summaries.score 80 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", + // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result with search on "dataset" @@ -339,12 +364,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); assertEquals( - "80", + "100", collections.getBody().getSearchAfter().get(1), "search_after 2 value" ); assertEquals( - "str:bc55eff4-7596-3565-e044-00144fdd4fa6", + "str:bf287dfe-9ce4-4969-9c59-51c39ea4d011", collections.getBody().getSearchAfter().get(2), "search_after 3 value" ); @@ -356,7 +381,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { String.format("'%s|| %s || %s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), - "bc55eff4-7596-3565-e044-00144fdd4fa6"), + "bf287dfe-9ce4-4969-9c59-51c39ea4d011"), HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -375,7 +400,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // The search after give you the value to go to next batch assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); assertEquals( - "str:19da2ce7-138f-4427-89de-a50c724f5f54", + "str:5c418118-2581-4936-b6fd-d6bedfe74f62", collections.getBody().getSearchAfter().get(2), "Search after 2 value" ); diff --git a/server/src/test/resources/databag/19da2ce7-138f-4427-89de-a50c724f5f54.json b/server/src/test/resources/databag/19da2ce7-138f-4427-89de-a50c724f5f54.json index 7349641c..74116701 100644 --- a/server/src/test/resources/databag/19da2ce7-138f-4427-89de-a50c724f5f54.json +++ b/server/src/test/resources/databag/19da2ce7-138f-4427-89de-a50c724f5f54.json @@ -28,7 +28,7 @@ ] }, "summaries": { - "score": 100, + "score": 106, "status": "completed", "credits": [ "Australia’s Integrated Marine Observing System (IMOS) is enabled by the National Collaborative Research Infrastructure Strategy (NCRIS). It is operated by a consortium of institutions as an unincorporated joint venture, with the University of Tasmania as Lead Agent.", diff --git a/server/src/test/resources/databag/5c418118-2581-4936-b6fd-d6bedfe74f62.json b/server/src/test/resources/databag/5c418118-2581-4936-b6fd-d6bedfe74f62.json index 22d740a5..c4975058 100644 --- a/server/src/test/resources/databag/5c418118-2581-4936-b6fd-d6bedfe74f62.json +++ b/server/src/test/resources/databag/5c418118-2581-4936-b6fd-d6bedfe74f62.json @@ -244,7 +244,7 @@ } ], "summaries": { - "score": 100, + "score": 90, "dataset_provider": null, "dataset_group": "aodn", "proj:geometry": { diff --git a/server/src/test/resources/databag/7709f541-fc0c-4318-b5b9-9053aa474e0e.json b/server/src/test/resources/databag/7709f541-fc0c-4318-b5b9-9053aa474e0e.json index 25d814ca..e4ec40dc 100644 --- a/server/src/test/resources/databag/7709f541-fc0c-4318-b5b9-9053aa474e0e.json +++ b/server/src/test/resources/databag/7709f541-fc0c-4318-b5b9-9053aa474e0e.json @@ -28,7 +28,7 @@ ] }, "summaries": { - "score": 95, + "score": 60, "status": "completed", "credits": [ "Australian Climate Change Science Program", From b1e15743ada8547357d7532e64243565f7ed7472 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 11 Nov 2025 11:26:51 +1100 Subject: [PATCH 08/10] update test samples and fix failed tests --- .../au/org/aodn/ogcapi/server/features/RestApiTest.java | 8 ++++---- .../databag/bc55eff4-7596-3565-e044-00144fdd4fa6.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 75c9751e..c357e1f5 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -71,7 +71,7 @@ public void verifyCorrectInternalPagingLargeData() throws IOException { "516811d7-cd1e-207a-e0440003ba8c79dd.json", // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - // set test summaries.score 80 + // set test summaries.score 50 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); @@ -120,7 +120,7 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { "516811d7-cd1e-207a-e0440003ba8c79dd.json", // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - // set test summaries.score 80 + // set test summaries.score 50 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); @@ -219,7 +219,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { "516811d7-cd1e-207a-e0440003ba8c79dd.json", // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - // set test summaries.score 80 + // set test summaries.score 50 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); @@ -333,7 +333,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { "516811d7-cd1e-207a-e0440003ba8c79dd.json", // set test summaries.score 60 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", - // set test summaries.score 80 + // set test summaries.score 50 "bc55eff4-7596-3565-e044-00144fdd4fa6.json", // set test summaries.score 100 "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); diff --git a/server/src/test/resources/databag/bc55eff4-7596-3565-e044-00144fdd4fa6.json b/server/src/test/resources/databag/bc55eff4-7596-3565-e044-00144fdd4fa6.json index 3c4284c3..6529575a 100644 --- a/server/src/test/resources/databag/bc55eff4-7596-3565-e044-00144fdd4fa6.json +++ b/server/src/test/resources/databag/bc55eff4-7596-3565-e044-00144fdd4fa6.json @@ -30,7 +30,7 @@ ] }, "summaries": { - "score": 80, + "score": 50, "status": "", "scope": { "code": "nonGeographicDataset", From 3f371593e6865f75d09ca15b068ae1e45fb189e3 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Mon, 24 Nov 2025 14:46:56 +1100 Subject: [PATCH 09/10] move useScriptScore logic into searchCollectionBy function --- .../server/core/service/ElasticSearch.java | 14 ++------------ .../server/core/service/ElasticSearchBase.java | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index ff76b4ac..9187755e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -211,8 +211,7 @@ protected ElasticSearchBase.SearchResult searchCollectionsB null, createSortOptions(sortBy, CQLFields.class), null, - null, - false); + null); } @Override @@ -339,14 +338,6 @@ public ElasticSearchBase.SearchResult searchByParameters(Li .toList(); } - boolean useScriptScore = false; - if (sortBy != null && !sortBy.isEmpty()) { - // only use script_score if sortby contains "score" and should field is not empty - if (sortBy.toLowerCase().contains("score") && should != null && !should.isEmpty()) { - useScriptScore = true; - } - } - return searchCollectionBy( null, should, @@ -355,8 +346,7 @@ public ElasticSearchBase.SearchResult searchByParameters(Li searchAfter, createSortOptions(sortBy, CQLFields.class), score, - maxSize, - useScriptScore + maxSize ); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index eb495eb9..b712d1c8 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -4,6 +4,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; import au.org.aodn.ogcapi.server.core.model.enumeration.StacBasicField; +import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.*; import co.elastic.clients.elasticsearch._types.aggregations.*; @@ -156,8 +157,7 @@ protected SearchResult searchCollectionBy(final List final List searchAfter, final List sortOptions, final Double score, - final Long maxSize, - final boolean useScriptScore) { + final Long maxSize) { Supplier builderSupplier = () -> { SearchRequest.Builder builder = new SearchRequest.Builder(); builder.index(indexName) @@ -169,7 +169,17 @@ protected SearchResult searchCollectionBy(final List // use script score if search with text, in such case, the final score depends on both relevance and metadata quality // put query in script block + // determine to use script score block or not + boolean useScriptScore = false; + if (sortOptions != null && !sortOptions.isEmpty()) { + // only use script_score if sortby contains "score" and should field is not empty + if (should != null && !should.isEmpty()) { + useScriptScore = true; + } + } + if (useScriptScore) { + String summaryScore = StacSummeries.Score.searchField; builder.query(q -> q.scriptScore(ss -> ss // to get the original _score from ELasticsearch .query(bq -> bq.bool(createBoolQueryForProperties(queries, should, filters))) @@ -178,8 +188,8 @@ protected SearchResult searchCollectionBy(final List .source( // Step 1: Retrieve internal quality score from summaries.score field // Default to 0 if field doesn't exist or is empty - "double internalScore = doc.containsKey('summaries.score') && " + - "!doc['summaries.score'].empty ? doc['summaries.score'].value : 0.0; " + + "double internalScore = doc.containsKey('"+summaryScore+"') && " + + "!doc['"+summaryScore+"'].empty ? doc['"+summaryScore+"'].value : 0.0; " + // Step 2: Normalize internal score to 0-1 range // Assuming summaries.score is in range 0-106 From cd851bbf74ce692e62390bf44acca5c977b7e362 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Mon, 24 Nov 2025 17:07:15 +1100 Subject: [PATCH 10/10] use simplified code --- .../ogcapi/server/core/service/CacheNoLandGeometry.java | 3 +-- .../aodn/ogcapi/server/core/service/ElasticSearch.java | 5 ----- .../ogcapi/server/core/service/ElasticSearchBase.java | 9 ++------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java index befb5d94..c032c0ac 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java @@ -38,8 +38,7 @@ public Map getAllNoLandGeometry() { null, null, null, - null, - false); + null); return result.collections .stream() diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 9187755e..15779168 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -240,11 +240,6 @@ public ElasticSearchBase.SearchResult searchAllCollections( return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } - /*** - * This function is used for searching by user input query. In such case, the sorting is from two impactors: the relavance score (powered by Elasticsearch) in the `._score` field, - * and the importance score (powered by IMOS internal ranking, which has a maximum value of 106). - * The final score is relavance_score * (importance_score / max(importance_score)) - */ @Override public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index b712d1c8..9cbb44de 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -170,13 +170,8 @@ protected SearchResult searchCollectionBy(final List // use script score if search with text, in such case, the final score depends on both relevance and metadata quality // put query in script block // determine to use script score block or not - boolean useScriptScore = false; - if (sortOptions != null && !sortOptions.isEmpty()) { - // only use script_score if sortby contains "score" and should field is not empty - if (should != null && !should.isEmpty()) { - useScriptScore = true; - } - } + // only use script_score if sortby contains "score" and should field is not empty + boolean useScriptScore = (sortOptions != null && !sortOptions.isEmpty()) && (should != null && !should.isEmpty()); if (useScriptScore) { String summaryScore = StacSummeries.Score.searchField;