From 976341b021647aa821672145de8e7b38c4024058 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 23 Dec 2025 12:27:21 +1100 Subject: [PATCH 1/4] Checkpoint get ncwms layer info --- .../server/core/model/ogc/wms/LayerInfo.java | 2 + .../core/model/ogc/wms/NcWmsLayerInfo.java | 70 ++++ .../core/service/wms/WmsDefaultParam.java | 1 + .../server/core/service/wms/WmsServer.java | 81 ++++- .../ogcapi/server/features/RestServices.java | 1 + server/src/main/resources/application.yaml | 7 + .../core/service/wms/WmsServerTest.java | 314 +++++++++--------- 7 files changed, 303 insertions(+), 173 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/NcWmsLayerInfo.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/LayerInfo.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/LayerInfo.java index 34d996c9..40a38aae 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/LayerInfo.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/LayerInfo.java @@ -48,4 +48,6 @@ public class LayerInfo { @JacksonXmlProperty(localName = "Style") private Style style; + + protected NcWmsLayerInfo ncWmsLayerInfo; } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/NcWmsLayerInfo.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/NcWmsLayerInfo.java new file mode 100644 index 00000000..99da09c4 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wms/NcWmsLayerInfo.java @@ -0,0 +1,70 @@ +package au.org.aodn.ogcapi.server.core.model.ogc.wms; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class NcWmsLayerInfo { + + @JsonProperty("title") + private String title; + + @JsonProperty("abstract") + private String abstractText; + + @JsonProperty("units") + private String units; + + @JsonProperty("bbox") + private List bbox; + + @JsonProperty("scales") + private List scales; + + @JsonProperty("palettes") + private List palettes; + + @JsonProperty("defaultPalette") + private String defaultPalette; + + @JsonProperty("timeAxis") + private List timeAxis; + + @JsonProperty("elevationAxis") + private List elevationAxis; + + @JsonProperty("scaleRange") + private List scaleRange; + + @JsonProperty("datesWithData") + private Map>> datesWithData; + + @JsonProperty("numColorBands") + private Integer numColorBands; + + @JsonProperty("supportedStyles") + private List supportedStyles; + + @JsonProperty("moreInfo") + private String moreInfo; + + @JsonProperty("timeAxisUnits") + private String timeAxisUnits; + + @JsonProperty("nearestTimeIso") + private String nearestTimeIso; + + @JsonProperty("logScaling") + private boolean logScaling; +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsDefaultParam.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsDefaultParam.java index 52260815..19447dd4 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsDefaultParam.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsDefaultParam.java @@ -22,6 +22,7 @@ public class WmsDefaultParam { private Map wms; private Map ncwms; + private Map ncmetadata; private Map descLayer; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java index 703f637b..c71dcc60 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java @@ -5,16 +5,14 @@ import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import au.org.aodn.ogcapi.server.core.model.ogc.wfs.DownloadableFieldModel; -import au.org.aodn.ogcapi.server.core.model.ogc.wms.DescribeLayerResponse; -import au.org.aodn.ogcapi.server.core.model.ogc.wms.FeatureInfoResponse; -import au.org.aodn.ogcapi.server.core.model.ogc.wms.GetCapabilitiesResponse; -import au.org.aodn.ogcapi.server.core.model.ogc.wms.LayerInfo; +import au.org.aodn.ogcapi.server.core.model.ogc.wms.*; import au.org.aodn.ogcapi.server.core.service.ElasticSearchBase; import au.org.aodn.ogcapi.server.core.service.Search; import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer; import au.org.aodn.ogcapi.server.core.util.RestTemplateUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.extern.slf4j.Slf4j; @@ -29,6 +27,7 @@ import java.math.BigDecimal; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -56,6 +55,9 @@ public class WmsServer { @Autowired protected WmsDefaultParam wmsDefaultParam; + @Autowired + protected ObjectMapper objectMapper; + @Lazy @Autowired protected WmsServer self; @@ -114,8 +116,8 @@ else if(wfsUrlComponents.getQueryParams().get("CQL_FILTER") != null) { if (range.size() == 2) { // Due to no standard name, we try our best to guess if 2 dateTime field, range mean we found start/end date String[] d = request.getDatetime().split("/"); - String guess1 = target.get(0).getName(); - String guess2 = target.get(1).getName(); + String guess1 = range.get(0).getName(); + String guess2 = range.get(1).getName(); if ((guess1.contains("start") || guess1.contains("min")) && (guess2.contains("end") || guess2.contains("max"))) { String timeCql = String.format("CQL_FILTER=%s >= %s AND %s <= %s", guess1, d[0], guess2, d[1]); @@ -174,6 +176,7 @@ protected List createMapQueryUrl(String url, String uuid, FeatureRequest param.putAll(wmsDefaultParam.getWms()); } else if (pathSegments.get(pathSegments.size() - 1).equalsIgnoreCase("ncwms")) { param.putAll(wmsDefaultParam.getNcwms()); + param.put("TIME", request.getDatetime()); } // Now we add the missing argument from the request @@ -215,12 +218,23 @@ protected List createMapQueryUrl(String url, String uuid, FeatureRequest builder.queryParam(key, value); } }); - // Cannot set cql in param as it contains value like "/" which is not allow in UriComponent checks - // but server must use "/" in param and cannot encode it to %2F, so to avoid exception in the - // build() call, we append the cql after the construction. - String target = String.join("&", builder.build().toUriString(), createCQLFilter(uuid, request)); - log.debug("Url to wms geoserver {}", target); - urls.add(target); + if (pathSegments.get(pathSegments.size() - 1).equalsIgnoreCase("ncwms")) { + // ncWMS (including GeoServer extension) does not support CQL_FILTER. + // It focuses on NetCDF gridded data with parameters like TIME, ELEVATION, COLORSCALERANGE, + // STYLES (palettes), NUMCOLORBANDS. CQL_FILTER is a GeoServer vendor parameter for vector + // filtering, not implemented in ncWMS. So we only add CQL if it is WMS + String target = builder.build().toUriString(); + log.debug("Url to ncWms geoserver {}", target); + urls.add(target); + } + else { + // Cannot set cql in param as it contains value like "/" which is not allow in UriComponent checks + // but server must use "/" in param and cannot encode it to %2F, so to avoid exception in the + // build() call, we append the cql after the construction. + String target = String.join("&", builder.build().toUriString(), createCQLFilter(uuid, request)); + log.debug("Url to wms geoserver {}", target); + urls.add(target); + } return urls; } @@ -494,7 +508,13 @@ public byte[] getMapTile(String collectionId, FeatureRequest request) throws URI log.debug("map tile request for layer name {} url {} ", request.getLayerName(), url); ResponseEntity response = restTemplateUtils.handleRedirect(url, restTemplate.exchange(url, HttpMethod.GET, pretendUserEntity, byte[].class), byte[].class, pretendUserEntity); if (response.getStatusCode().is2xxSuccessful()) { - return response.getBody(); + if (response.getHeaders().getContentType() != null && response.getHeaders().getContentType().getType().equals("image")) { + return response.getBody(); + } + else { + // Something wrong from the server likely syntax error + throw new URISyntaxException(response.getBody() != null ? new String(response.getBody(), StandardCharsets.UTF_8) : "", url); + } } } } @@ -602,16 +622,43 @@ public List getCapabilitiesLayers(String collectionId, FeatureRequest if(filteredLayers.isEmpty() && request.getLayerName() != null) { DescribeLayerResponse dr = describeLayer(collectionId, request); if(dr != null) { - // That means at least layer is valid just not operational - return List.of( + // That means at least layer is valid just not works with wfs, we should keep the + // original layername instead showing the lookup name as it can be different from + // what is mentioned in the metadata which people get confused. + filteredLayers = List.of( LayerInfo.builder() - .name(dr.getLayerDescription().getName()) - .title(dr.getLayerDescription().getName()) + .name(request.getLayerName()) + .title(request.getLayerName()) .queryable("0") .build() ); } } + + // Special case for NCWMS layer where we need to call GetMetadata to find the related points for gridded data + if(mapServerUrl.get().contains("/ncwms")) { + filteredLayers.forEach(layer -> { + // For each ncwms layer, we attach the metadata + UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(mapServerUrl.get()); + builder.scheme("https"); // Force https + + wmsDefaultParam.getNcmetadata().forEach((key, value) -> { + if (value != null) { + builder.queryParam(key, value); + } + }); + builder.queryParam("layerName", layer.getName()); + + ResponseEntity response = restTemplate.exchange(builder.toUriString(), HttpMethod.GET, pretendUserEntity, String.class); + if(response.getStatusCode().is2xxSuccessful()) { + try { + layer.setNcWmsLayerInfo(objectMapper.readValue(response.getBody(), NcWmsLayerInfo.class)); + } catch (JsonProcessingException e) { + // Save to ignore + } + } + }); + } log.debug("Returning layers {}", filteredLayers); return filteredLayers; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index 702e2626..2917ea7d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -78,6 +78,7 @@ public ResponseEntity getWmsMapFeature(String collectionId, public ResponseEntity getWmsMapTile(String collectionId, FeatureRequest request) { try { return ResponseEntity.ok() + .contentType(MediaType.IMAGE_PNG) .body(wmsServer.getMapTile(collectionId, request)); } catch (Throwable e) { throw new RuntimeException(e); diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 2f9563aa..54a3b29e 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -91,8 +91,15 @@ wms-default-param: STYLES: "" QUERYABLE: "true" CRS: "EPSG:3857" + NUMCOLORBANDS: "253" WIDTH: 256 HEIGHT: 256 + ncmetadata: + SERVICE: "ncwms" + REQUEST: "GetMetadata" + VERSION: "1.3.0" + # Must be small letter item !!! + item: "layerDetails" allow-id: # Full list - 4402cb50-e20a-44ee-93e6-4728259250d2 - ae86e2f5-eaaf-459e-a405-e654d85adb9c diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServerTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServerTest.java index 387cd458..cd9581f6 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServerTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServerTest.java @@ -252,13 +252,14 @@ public void verifyHandleServiceExceptionReportCorrect() { .build() ); - String r = "\n" + - "\n" + - "\n" + - " \n" + - " srs_ghrsst_l4_gamssa_url/: no such layer on this server\n" + - "\n" + - ""; + String r = """ + + + + + srs_ghrsst_l4_gamssa_url/: no such layer on this server + + """; when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), eq(entity), eq(String.class))) .thenReturn(ResponseEntity.ok(r)); @@ -275,13 +276,14 @@ public void verifyHandleServiceExceptionReportCorrect() { @Test public void verifyParseCorrect() throws JsonProcessingException { DescribeLayerResponse value = wmsServer.xmlMapper.readValue( - "\n" + - "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - "", DescribeLayerResponse.class); + """ + + + + + + + """, DescribeLayerResponse.class); assertEquals("imos:srs_ghrsst_l4_gamssa_url", value.getLayerDescription().getName()); assertEquals("https://geoserver-123.aodn.org.au/geoserver/wfs?", value.getLayerDescription().getWfs()); @@ -289,10 +291,9 @@ public void verifyParseCorrect() throws JsonProcessingException { } /** * Test with only one dateTime field in the describe layer - * @throws JsonProcessingException - Not expected */ @Test - public void verifyCreateCQLSingleDateTime() throws JsonProcessingException { + public void verifyCreateCQLSingleDateTime() { String uuid = "uuid1"; String layer = "imos:srs_ghrsst_l4_gamssa_url"; FeatureRequest request = FeatureRequest.builder() @@ -316,23 +317,24 @@ public void verifyCreateCQLSingleDateTime() throws JsonProcessingException { ); // This sample contains 1 dateTime field String value = - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - ""; + """ + + + + + + + + + + + + + + + + + """; when(search.searchCollections(eq(uuid))) .thenReturn(stac); @@ -350,10 +352,9 @@ public void verifyCreateCQLSingleDateTime() throws JsonProcessingException { } /** * Test with only one dateTime field in the describe layer with predefined cql - * @throws JsonProcessingException - Not expected */ @Test - public void verifyCreateCQLSingleDateTimeWithCQL() throws JsonProcessingException { + public void verifyCreateCQLSingleDateTimeWithCQL() { String uuid = "uuid1"; String layer = "imos:srs_ghrsst_l4_gamssa_url"; FeatureRequest request = FeatureRequest.builder() @@ -377,23 +378,24 @@ public void verifyCreateCQLSingleDateTimeWithCQL() throws JsonProcessingExceptio ); // This sample contains 1 dateTime field String value = - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - ""; + """ + + + + + + + + + + + + + + + + + """; when(search.searchCollections(eq(uuid))) .thenReturn(stac); @@ -411,10 +413,9 @@ public void verifyCreateCQLSingleDateTimeWithCQL() throws JsonProcessingExceptio } /** * Test with only two or more dateTime field in the describe layer with predefined cql - * @throws JsonProcessingException - Not expected */ @Test - public void verifyCreateCQLMultiDateTimeWithCQL() throws JsonProcessingException { + public void verifyCreateCQLMultiDateTimeWithCQL() { String uuid = "uuid1"; String layer = "aatams_sattag_qc_ctd_profile_map"; FeatureRequest request = FeatureRequest.builder() @@ -438,56 +439,57 @@ public void verifyCreateCQLMultiDateTimeWithCQL() throws JsonProcessingException ); // This sample contains 1 dateTime field String value = - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - ""; + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """; when(search.searchCollections(eq(uuid))) .thenReturn(stac); @@ -505,10 +507,9 @@ public void verifyCreateCQLMultiDateTimeWithCQL() throws JsonProcessingException } /** * Test where dateTime is a range start_xxx end_xxx - * @throws JsonProcessingException - Not expected */ @Test - public void verifyCreateCQLRangeDateTimeWithCQL() throws JsonProcessingException { + public void verifyCreateCQLRangeDateTimeWithCQL() { String uuid = "uuid1"; String layer = "aatams_sattag_qc_ctd_profile_map"; FeatureRequest request = FeatureRequest.builder() @@ -532,56 +533,57 @@ public void verifyCreateCQLRangeDateTimeWithCQL() throws JsonProcessingException ); // This sample contains 1 dateTime field String value = - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - ""; + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """; when(search.searchCollections(eq(uuid))) .thenReturn(stac); From 21157a87c4ba908db7228e5714fe27866f40190f Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 24 Dec 2025 10:16:08 +1100 Subject: [PATCH 2/4] Minor fix to handle null case --- .../au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java index c71dcc60..26986807 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java @@ -176,7 +176,9 @@ protected List createMapQueryUrl(String url, String uuid, FeatureRequest param.putAll(wmsDefaultParam.getWms()); } else if (pathSegments.get(pathSegments.size() - 1).equalsIgnoreCase("ncwms")) { param.putAll(wmsDefaultParam.getNcwms()); - param.put("TIME", request.getDatetime()); + if(request.getDatetime() != null) { + param.put("TIME", request.getDatetime()); + } } // Now we add the missing argument from the request From c63fb73e80ad226538f9f428a8817ee6a7bfa4a4 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 24 Dec 2025 11:23:38 +1100 Subject: [PATCH 3/4] Previous check is too wide --- .../ogcapi/server/core/service/wfs/WfsServer.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java index 32da6b1c..8eae715b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java @@ -183,10 +183,11 @@ public Optional getFeatureServerUrlByTitleOrQueryParam(String collection /** * Fuzzy match utility to compare layer names, ignoring namespace prefixes * For example: "underway:nuyina_underway_202122020" matches "nuyina_underway_202122020" + * For example: "abc/cde" matches "abc" * * @param text1 - First text to compare * @param text2 - Second text to compare - * @return true if texts match (after removing namespace prefix) + * @return true if texts match (after removing namespace prefix) and subfix */ protected boolean roughlyMatch(String text1, String text2) { if (text1 == null || text2 == null) { @@ -197,13 +198,11 @@ protected boolean roughlyMatch(String text1, String text2) { String normalized1 = text1.contains(":") ? text1.substring(text1.indexOf(":") + 1) : text1; String normalized2 = text2.contains(":") ? text2.substring(text2.indexOf(":") + 1) : text2; - if (normalized1.length() < normalized2.length()) { - // Swap the text so that compare startsWith using longer text. - String temp = normalized1; - normalized1 = normalized2; - normalized2 = temp; - } - return normalized1.startsWith(normalized2); + // Remove "/" and anything follows + normalized1 = normalized1.split("/")[0]; + normalized2 = normalized2.split("/")[0]; + + return normalized1.equals(normalized2); } /** * Extract typename from WFS URL query parameters From ca2261b9886300913dab2f3268ff003baa9d6dd4 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 24 Dec 2025 11:27:16 +1100 Subject: [PATCH 4/4] Previous check is too wide --- .../au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java index ca18872d..360e42c0 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java @@ -135,6 +135,6 @@ void primaryTitleMatch_filtersPreferAodnMapLayers() { List info = server.filterLayersByWfsLinks("id", layers); assertEquals(1, info.size(), "Layer count match"); - assertEquals(layers.get(1), info.get(0), "Layer layer:test_layer_aodn_map found"); + assertEquals(layers.get(0), info.get(0), "Layer layer:test_layer_aodn_map found"); } }