From 33ad6bd5205e43645a76cc12702341aaa30a6e7f Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Mon, 6 Sep 2021 16:57:39 +0100 Subject: [PATCH 1/3] Add geospatial quicklook processing support --- HowToDeploy.txt | 29 ++ calvalus-processing/pom.xml | 7 + .../calvalus/processing/JobConfigNames.java | 5 + .../processing/analysis/GeoServer.java | 266 ++++++++++++++++++ .../processing/analysis/QLMapper.java | 228 ++++++++++++++- .../processing/analysis/Quicklooks.java | 8 + .../processing/l2/L2FormattingMapper.java | 38 ++- .../production/hadoop/QLProductionType.java | 12 +- 8 files changed, 570 insertions(+), 23 deletions(-) create mode 100644 calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java diff --git a/HowToDeploy.txt b/HowToDeploy.txt index b08be9b15..b41c6087b 100644 --- a/HowToDeploy.txt +++ b/HowToDeploy.txt @@ -192,5 +192,34 @@ cp -r /home/cvop/calbfg-patch/* webapps/calbfg rm -rf webapps/calbfg/staging/ mv temp/staging webapps/calbfg/ +=========================================================================== +GeoServer - Calvalus production +------------------------------- +Calvalus supports the automated publishing of both regular GeoTIFF (geotiff imageType) and Cloud +Optimized GeoTIFF (cog imageType) images to GeoServer using the existing quicklooks production feature. + +The GeoServer data store name used by the upload service is automatically generated using the product +name. Any existing data store with this product name within the workspace is replaced. Likewise, +the GeoServer layer name used by the upload service is also generated using the same product name. Any +existing layer with this name within the workspace is also replaced. The upload service uses the +default GeoServer raster style. Once data is published to GeoServer, the data product is automatically +accessible from GeoServer via Web Mapping Service (WMS). + +The Calvalus administrator needs to configure these GeoServer quicklooks settings in calvalus.properties +to enable GeoServer - Calvalus production support: + +# Quicklooks upload handler. Note: only geoserver supported. Mandatory. +calvalus.ql.upload.handler = geoserver + +# Quicklooks upload URL. Mandatory. Example: +calvalus.ql.upload.URL = http://localhost:8080/geoserver/rest/ + +# Username and password for the GeoServer upload URL service. Optional. Example: +calvalus.ql.upload.username = admin +calvalus.ql.upload.password = geoserver + +# GeoServer workspace to use. Mandatory. Example: +calvalus.ql.upload.workspace = calmar + diff --git a/calvalus-processing/pom.xml b/calvalus-processing/pom.xml index a685ab7a0..ed2fb3fe1 100644 --- a/calvalus-processing/pom.xml +++ b/calvalus-processing/pom.xml @@ -225,5 +225,12 @@ org.renjin renjin-core + + + + org.gdal + gdal + 3.3.0 + diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/JobConfigNames.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/JobConfigNames.java index f24a2b141..b652732f3 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/JobConfigNames.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/JobConfigNames.java @@ -57,6 +57,11 @@ public interface JobConfigNames { String CALVALUS_OUTPUT_QUICKLOOKS = "calvalus.output.quicklooks"; String CALVALUS_QUICKLOOK_PARAMETERS = "calvalus.ql.parameters"; + String CALVALUS_QUICKLOOK_UPLOAD_HANDLER = "calvalus.ql.upload.handler"; + String CALVALUS_QUICKLOOK_UPLOAD_URL = "calvalus.ql.upload.URL"; + String CALVALUS_QUICKLOOK_UPLOAD_USERNAME = "calvalus.ql.upload.username"; + String CALVALUS_QUICKLOOK_UPLOAD_PASSWORD = "calvalus.ql.upload.password"; + String CALVALUS_QUICKLOOK_UPLOAD_WORKSPACE = "calvalus.ql.upload.workspace"; String CALVALUS_REQUEST_SIZE_LIMIT = "calvalus.requestSizeLimit"; diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java new file mode 100644 index 000000000..a7eaa8512 --- /dev/null +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2019 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.processing.analysis; + +import com.bc.calvalus.commons.CalvalusLogger; +import com.bc.calvalus.processing.JobConfigNames; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Base64; +import java.util.logging.Logger; + +/** + * GeoServer REST API handler + * + * @author Declan + */ +public class GeoServer { + + private static final Logger LOGGER = CalvalusLogger.getLogger(); + + private final TaskAttemptContext context; + private String basicAuth = ""; + + public GeoServer(TaskAttemptContext context ) { + this.context = context; + } + + public void uploadImage(InputStream inputStream, String imageName) throws IOException { + LOGGER.info(String.format("Uploading product image '%s.tiff' to GeoServer.", imageName)); + + String geoserverRestURL = context.getConfiguration().get(JobConfigNames.CALVALUS_QUICKLOOK_UPLOAD_URL); + if (geoserverRestURL == null || geoserverRestURL.isEmpty()) { + throw new IllegalArgumentException("geoserverRestURL is empty"); + } + geoserverRestURL = geoserverRestURL.trim(); + while (geoserverRestURL.endsWith("/")) { + geoserverRestURL = geoserverRestURL.substring(0, geoserverRestURL.length() - 1); + } + + String username = context.getConfiguration().get(JobConfigNames.CALVALUS_QUICKLOOK_UPLOAD_USERNAME); + String password = context.getConfiguration().get(JobConfigNames.CALVALUS_QUICKLOOK_UPLOAD_PASSWORD); + if (username != null && !username.isEmpty() && password != null) { + String userpass = username + ":" + password; + this.basicAuth = "Basic " + new String(Base64.getEncoder().encode(userpass.getBytes())); + } + + String workspace = context.getConfiguration().get(JobConfigNames.CALVALUS_QUICKLOOK_UPLOAD_WORKSPACE); + if (workspace == null || workspace.isEmpty()) { + throw new IllegalArgumentException("workspace is empty"); + } + + String store = imageName; + String layer = imageName; + + // other functionality for future (not required at present) + //getCoverageStore(geoserverRestURL, workspace, store); + //deleteCoverageStore(geoserverRestURL, workspace, store); + //putLayerStyle(geoserverRestURL, workspace, layer, style); + + putCoverageStoreSingleGeoTiff(inputStream, geoserverRestURL, workspace, store, layer); + } + + /** + * Get a coverage store named {storeName} in the {workspace} workspace + * + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param store the name of the store to be retrieved + */ + private void getCoverageStore(String geoserverRestURL, String workspace, String store) throws IOException { + String getCoverageStoreURL = String.format("%s/workspaces/%s/coveragestores/%s", geoserverRestURL, workspace, store); + HttpURLConnection conn = null; + try { + LOGGER.info(String.format("Getting coverage store '%s' in workspace '%s' on GeoServer...", store, workspace)); + LOGGER.info(String.format("GeoServer URL: %s", getCoverageStoreURL)); + URL url = new URL(getCoverageStoreURL); + conn = (HttpURLConnection) url.openConnection(); + if (!this.basicAuth.isEmpty()) { + conn.setRequestProperty("Authorization", basicAuth); + } + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestProperty("Content-Type", "application/xml"); + conn.setRequestProperty("charset", "utf-8"); + logResponse(conn); + LOGGER.info(String.format("Finished getting coverage store '%s' in workspace '%s' on GeoServer", store, workspace)); + } catch (IOException e) { + LOGGER.warning(String.format("GeoServer error: %s", e.getMessage())); + logResponse(conn, true); + } finally { + if (conn != null) + conn.disconnect(); + } + } + + /** + * Delete a coverage store named {storeName} in the {workspace} workspace + * + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param store the name of the store to be deleted + */ + private void deleteCoverageStore(String geoserverRestURL, String workspace, String store) throws IOException { + String deleteCoverageStoreURL = String.format("%s/workspaces/%s/coveragestores/%s?recurse=true&purge=all", geoserverRestURL, workspace, store); + HttpURLConnection conn = null; + try { + LOGGER.info(String.format("Deleting coverage store '%s' in workspace '%s' on GeoServer...", store, workspace)); + LOGGER.info(String.format("GeoServer URL: %s", deleteCoverageStoreURL)); + URL url = new URL(deleteCoverageStoreURL); + conn = (HttpURLConnection) url.openConnection(); + if (!this.basicAuth.isEmpty()) { + conn.setRequestProperty("Authorization", basicAuth); + } + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestMethod("DELETE"); + logResponse(conn); + LOGGER.info(String.format("Finished deleting coverage store '%s' in workspace '%s' on GeoServer", store, workspace)); + } catch (IOException e) { + LOGGER.warning(String.format("GeoServer error: %s", e.getMessage())); + logResponse(conn, true); + } finally { + if (conn != null) + conn.disconnect(); + } + } + + /** + * Creates or overwrites a single coverage store by uploading its GeoTIFF raster data file + * + * @param inputStream an InputStream to the GeoTIFF for upload + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param store the name of the store to be created or overwritten + * @param layer the name of the new layer + */ + private void putCoverageStoreSingleGeoTiff(InputStream inputStream, String geoserverRestURL, String workspace, String store, String layer) throws IOException { + String putCoverageStoreURL = String.format("%s/workspaces/%s/coveragestores/%s/file.geotiff?coverageName=%s", geoserverRestURL, workspace, store, layer); + byte[] geoTiffPayload = IOUtils.toByteArray(inputStream); + HttpURLConnection conn = null; + DataOutputStream out = null; + try { + LOGGER.info(String.format("Uploading geoTIFF layer '%s' to coverage store '%s' in workspace '%s' on GeoServer...", layer, store, workspace)); + LOGGER.info(String.format("GeoServer URL: %s", putCoverageStoreURL)); + LOGGER.info(String.format("GeoTIFF size (bytes): %d", geoTiffPayload.length)); + URL url = new URL(putCoverageStoreURL); + conn = (HttpURLConnection) url.openConnection(); + if (!this.basicAuth.isEmpty()) { + conn.setRequestProperty("Authorization", basicAuth); + } + conn.setDoInput(true); + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestProperty("Content-Type", "text/plain"); + conn.setRequestMethod("PUT"); + out = new DataOutputStream(conn.getOutputStream()); + out.write(geoTiffPayload); + out.flush(); + logResponse(conn); + LOGGER.info(String.format("Finished uploading geoTIFF layer '%s' to coverage store '%s' in workspace '%s' on GeoServer", layer, store, workspace)); + } catch (IOException e) { + LOGGER.warning(String.format("GeoServer error: %s", e.getMessage())); + logResponse(conn, true); + } finally { + if (out != null) + out.close(); + if (conn != null) + conn.disconnect(); + } + } + + /** + * Modify a layer's style + * + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param layer the name of the layer to modify + * @param style the default style to use for this layer + */ + private void putLayerStyle(String geoserverRestURL, String workspace, String layer, String style) throws IOException { + String payload = "" + style + ""; + String uploadStyleURL = String.format("%s/workspaces/%s/layers/%s", geoserverRestURL, workspace, layer); + HttpURLConnection conn = null; + DataOutputStream out = null; + try { + LOGGER.info(String.format("Updating layer '%s' to named style '%s' in workspace '%s' on GeoServer", layer, style, workspace)); + LOGGER.info(String.format("GeoServer URL: %s", uploadStyleURL)); + URL url = new URL(uploadStyleURL); + conn = (HttpURLConnection) url.openConnection(); + if (!this.basicAuth.isEmpty()) { + conn.setRequestProperty("Authorization", basicAuth); + } + conn.setDoInput(true); + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestProperty("Content-Type", "application/xml"); + conn.setRequestMethod("PUT"); + out = new DataOutputStream(conn.getOutputStream()); + out.writeBytes(payload); + out.flush(); + logResponse(conn); + LOGGER.info(String.format("Finished updating layer '%s' to named style '%s' in workspace '%s' on GeoServer", layer, style, workspace)); + } catch (IOException e) { + LOGGER.warning(String.format("GeoServer error: %s", e.getMessage())); + logResponse(conn, true); + } finally { + if (out != null) + out.close(); + if (conn != null) + conn.disconnect(); + } + } + + private void logResponse(HttpURLConnection conn) throws IOException { + logResponse(conn, false); + } + + private void logResponse(HttpURLConnection conn, boolean exception) throws IOException { + InputStream inputStream = null; + try { + int responseCode = conn.getResponseCode(); + if (exception) { + inputStream = conn.getErrorStream(); + } else { + inputStream = conn.getInputStream(); + } + + if (inputStream != null) { + ByteArrayOutputStream response = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + response.write(buffer, 0, length); + } + LOGGER.info(String.format("GeoServer HTTP response code: %d", responseCode)); + LOGGER.info(String.format("GeoServer HTTP response message: %s", response.toString())); + } + } finally { + if (inputStream != null) { + inputStream.close(); + } + } + } +} \ No newline at end of file diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java index 887db1d39..4f7b6878b 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java @@ -24,19 +24,49 @@ import com.bc.calvalus.processing.l2.L2FormattingMapper; import com.bc.ceres.core.ProgressMonitor; import com.bc.ceres.core.SubProgressMonitor; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.esa.snap.core.datamodel.GeoCoding; +import org.esa.snap.core.datamodel.GeoPos; +import org.esa.snap.core.datamodel.PixelPos; import org.esa.snap.core.datamodel.Product; +import org.esa.snap.core.gpf.GPF; import org.esa.snap.core.util.io.FileUtils; +import org.gdal.gdal.Band; +import org.gdal.gdal.Dataset; +import org.gdal.gdal.Driver; +import org.gdal.gdal.gdal; +import org.gdal.osr.SpatialReference; +import org.gdal.gdalconst.gdalconst; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.grid.GridCoverageFactory; +import org.geotools.coverage.grid.io.AbstractGridFormat; +//import org.geotools.coverage.grid.io.imageio.GeoToolsWriteParams; +import org.geotools.gce.geotiff.GeoTiffFormat; +import org.geotools.gce.geotiff.GeoTiffWriteParams; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.crs.DefaultGeographicCRS; +import org.opengis.coverage.grid.GridCoverageWriter; +import org.opengis.parameter.GeneralParameterValue; +import org.opengis.parameter.ParameterValueGroup; import javax.imageio.ImageIO; +import java.awt.image.Raster; import java.awt.image.RenderedImage; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.InputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; +//import java.util.Vector; import java.util.logging.Logger; /** @@ -61,18 +91,38 @@ public void run(Mapper.Context context) throws IOException, InterruptedException final String productName = FileUtils.getFilenameWithoutExtension(inputFileName); final Quicklooks.QLConfig[] configs = Quicklooks.get(context.getConfiguration()); for (Quicklooks.QLConfig config : configs) { - final String imageFileName; + final String imageBaseName; if (context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REGEX) != null && context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REPLACEMENT) != null) { if (configs.length == 1) { - imageFileName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName); + imageBaseName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName); } else { - imageFileName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName) + "_" + config.getBandName(); + imageBaseName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName) + "_" + config.getBandName(); } } else { - imageFileName = productName + "_" + config.getBandName(); + imageBaseName = productName + "_" + config.getBandName(); + } + createQuicklook(product, imageBaseName, context, config); + final String qlUploadHandler = context.getConfiguration().get(JobConfigNames.CALVALUS_QUICKLOOK_UPLOAD_HANDLER); + + if (config.isWmsEnabled()) { + if (qlUploadHandler == null || qlUploadHandler.isEmpty()) { + LOGGER.warning("No quicklook upload handler configured"); + } else if (qlUploadHandler.equalsIgnoreCase("geoserver")) { + if (isGeoTiff(config) || isCloudOptimizedGeoTIFF(config)) { + // upload geoTiff to GeoServer + LOGGER.info(String.format("Quicklook upload handler: %s", qlUploadHandler)); + GeoServer geoserver = new GeoServer(context); + String imageFilename = QLMapper.getImageFileName(imageBaseName, config); + InputStream inputStream = QLMapper.createInputStream(context, imageFilename); + geoserver.uploadImage(inputStream, imageBaseName); + } else { + LOGGER.warning(String.format("Quicklook image format '%s' not supported for GeoServer upload. Please use am image format such as GeoTIFF or COG instead.", config.getImageType())); + } + } else { + LOGGER.warning(String.format("Unknown quicklook upload handler: %s", qlUploadHandler)); + } } - createQuicklook(product, imageFileName, context, config); } } } finally { @@ -81,31 +131,191 @@ public void run(Mapper.Context context) throws IOException, InterruptedException } } - public static void createQuicklook(Product product, String imageFileName, Mapper.Context context, + public static void createQuicklook(Product product, String imageBaseName, Mapper.Context context, Quicklooks.QLConfig config) throws IOException, InterruptedException { // try { - RenderedImage quicklookImage = new QuicklookGenerator(context, product, config).createImage(); + RenderedImage quicklookImage = null; + String imageFileName = getImageFileName(imageBaseName, config); + if (isGeoTiff(config) || isCloudOptimizedGeoTIFF(config)) { + // for geoTiff and Cloud Optimized GeoTIFF images, we will force EPSG:4326 projection + // then render the image after re-projection + Map reprojParams = new HashMap(); + reprojParams.put("crs", "EPSG:4326"); + //reprojParams.put("noDataValue", 0d); + product = GPF.createProduct("Reproject", reprojParams, product); + quicklookImage = new QuicklookGenerator(context, product, config).createImage(); if (quicklookImage != null) { - OutputStream outputStream = createOutputStream(context, imageFileName + "." + config.getImageType()); + final int width = product.getSceneRasterWidth(); + final int height = product.getSceneRasterHeight(); + final GeoCoding geoCoding = product.getSceneGeoCoding(); + final PixelPos posA = new org.esa.snap.core.datamodel.PixelPos(0, 0); + final GeoPos geoPosA = geoCoding.getGeoPos(posA, null); + final PixelPos posB = new org.esa.snap.core.datamodel.PixelPos(width, height); + final GeoPos geoPosB = geoCoding.getGeoPos(posB, null); + + if (isCloudOptimizedGeoTIFF(config)) { + // use GDAL Java bindings API to create a Cloud Optimized GeoTIFF (COG) + gdal.AllRegister(); + Driver driverGeoTiff = gdal.GetDriverByName("GTiff"); + if (driverGeoTiff == null) { + throw new RuntimeException("Could not load GDAL GTiff driver required for Cloud Optimized GeoTIFF (COG)"); + } + Driver driverCOG = gdal.GetDriverByName("COG"); + if (driverCOG == null) { + throw new RuntimeException("Could not load GDAL COG driver required for Cloud Optimized GeoTIFF (COG)"); + } + Raster quicklookRaster = quicklookImage.getData(); + int numberOfRasterBands = quicklookRaster.getNumBands(); + String tmpImageFileName = getTmpImageFileName(imageBaseName, config); + + /* + // Add GDAL specific control parameters for the GTiff driver if required in future. Example: + final Vector geoTiffCreationOptions = new Vector<>(4); + imageCreationOptions.add("COMPRESS=JPEG"); + imageCreationOptions.add("TILED=FALSE"); + imageCreationOptions.add("SPARSE_OK=FALSE"); + imageCreationOptions.add("PROFILE=GeoTIFF"); + Dataset geoTiffDataset = driverGeoTiff.Create(tmpImageFileName, width, height, numberOfRasterBands, gdalconst.GDT_Byte, geoTiffCreationOptions); + */ + Dataset geoTiffDataset = driverGeoTiff.Create(tmpImageFileName, width, height, numberOfRasterBands, gdalconst.GDT_Byte); + + // add SRS to the GDAL dataset object + SpatialReference srs = new SpatialReference(); + srs.ImportFromEPSG(4326); + geoTiffDataset.SetSpatialRef(srs); + + // add Geotransform (north up image) to the GDAL dataset object + // assumes longitude is between -90 and 90, and latitude between -180 and 180 + double pixelSizeX = Math.abs(geoPosA.getLon() - geoPosB.getLon()) / (double) width; + double pixelSizeY = Math.abs(geoPosA.getLat() - geoPosB.getLat()) / (double) height; + double[] geotransform = new double[6]; + geotransform[0] = geoPosA.getLon(); // longitude value for the top left pixel of the raster + geotransform[1] = pixelSizeX; // pixel width + geotransform[2] = 0; // 0 + geotransform[3] = geoPosA.getLat(); // latitude value for the top left pixel of the raster + geotransform[4] = 0; // 0 + geotransform[5] = -pixelSizeY; // pixel height (negative value) + geoTiffDataset.SetGeoTransform(geotransform); + + // add image data to the GDAL dataset object, for each image band + int[] band = new int[(height * width)]; + for (int i = 0; i < numberOfRasterBands; i++) { + quicklookRaster.getSamples(0, 0, width, height, i, band); + Band outBand = geoTiffDataset.GetRasterBand(i + 1); + outBand.WriteRaster(0, 0, width, height, gdalconst.GDT_Int32, band); + outBand.FlushCache(); + } + geoTiffDataset.FlushCache(); + + // create and copy COG image file using the GeoTIFF dataset + Dataset cogDataset = driverCOG.CreateCopy(imageFileName, geoTiffDataset); + + // must do this cleanup now, to get all header and data to disk. + // Otherwise the COG image becomes corrupt + cogDataset.FlushCache(); + cogDataset.delete(); + geoTiffDataset.delete(); + + // copy local Cloud Optimized GeoTIFF file to the HDFS filesystem + FileSystem localFilesystem = FileSystem.getLocal(context.getConfiguration()); + FileSystem hdfsFilesystem = FileSystem.get(context.getConfiguration()); + Path path = getWorkOutputFilePath(context, imageFileName); + FileUtil.copy(localFilesystem, new Path(imageFileName), hdfsFilesystem, path, false, context.getConfiguration()); + } else if (isGeoTiff(config)) { + // use GeoTools to create a simple GeoTIFF + final GeoTiffFormat format = new GeoTiffFormat(); + final GeoTiffWriteParams wp = new GeoTiffWriteParams(); + //wp.setCompressionMode(GeoTiffWriteParams.MODE_EXPLICIT); + //wp.setCompressionType("LZW"); + //wp.setCompressionQuality(1.0F); + //wp.setTilingMode(GeoToolsWriteParams.MODE_EXPLICIT); + //wp.setTiling(256, 256); + final ParameterValueGroup params = format.getWriteParameters(); + params.parameter(AbstractGridFormat.GEOTOOLS_WRITE_PARAMS.getName().toString()).setValue(wp); + + OutputStream outputStream = createOutputStream(context, imageFileName); + LOGGER.info("outputStream: " + outputStream.toString()); + OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); + try { + ReferencedEnvelope envelope = + new ReferencedEnvelope( + geoPosA.getLon(), geoPosB.getLon(), + geoPosA.getLat(), geoPosB.getLat(), + DefaultGeographicCRS.WGS84 + ); + GridCoverageFactory factory = new GridCoverageFactory(); + GridCoverage2D gridCoverage2D = factory.create(imageFileName, quicklookImage, envelope); + GridCoverageWriter writer = format.getWriter(new BufferedOutputStream(pmOutputStream)); + writer.write(gridCoverage2D, params.values().toArray(new GeneralParameterValue[1])); + writer.dispose(); + } finally { + outputStream.close(); + } + } + } + } else { + // for non geoTiff and Cloud Optimized GeoTIFF images, we will not force EPSG:4326 projection, + // instead just render the image as it is + quicklookImage = new QuicklookGenerator(context, product, config).createImage(); + if (quicklookImage != null) { + OutputStream outputStream = createOutputStream(context, imageFileName); + LOGGER.info("outputStream: " + outputStream.toString()); OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); try { ImageIO.write(quicklookImage, config.getImageType(), pmOutputStream); } finally { outputStream.close(); } + } + } // } catch (Exception e) { // String msg = String.format("Could not create quicklook image '%s'.", config.getBandName()); // LOGGER.log(Level.WARNING, msg, e); // } } + public static String getImageFileName(String imageBaseName, Quicklooks.QLConfig qlConfig) { + if (isGeoTiff(qlConfig) || isCloudOptimizedGeoTIFF(qlConfig)) { + return imageBaseName + ".tiff"; + } else { + return imageBaseName + "." + qlConfig.getImageType(); + } + } + + public static String getTmpImageFileName(String imageBaseName, Quicklooks.QLConfig qlConfig) { + if (isGeoTiff(qlConfig) || isCloudOptimizedGeoTIFF(qlConfig)) { + return imageBaseName + "_tmp.tiff"; + } else { + return imageBaseName + "_tmp." + qlConfig.getImageType(); + } + } + + public static Path getWorkOutputFilePath(Mapper.Context context, String fileName) throws IOException, InterruptedException { + return new Path(FileOutputFormat.getWorkOutputPath(context), fileName); + } + + private static boolean isGeoTiff(Quicklooks.QLConfig qlConfig) { + return "geotiff".equalsIgnoreCase(qlConfig.getImageType()); + } + + private static boolean isCloudOptimizedGeoTIFF(Quicklooks.QLConfig qlConfig) { + return "cog".equalsIgnoreCase(qlConfig.getImageType()); + } + private static OutputStream createOutputStream(Mapper.Context context, String fileName) throws IOException, InterruptedException { - Path path = new Path(FileOutputFormat.getWorkOutputPath(context), fileName); + Path path = getWorkOutputFilePath(context, fileName); + LOGGER.info("createOutputStream: path = " + path); final FSDataOutputStream fsDataOutputStream = path.getFileSystem(context.getConfiguration()).create(path); return new BufferedOutputStream(fsDataOutputStream); } + public static InputStream createInputStream(Mapper.Context context, String fileName) throws IOException, InterruptedException { + Path path = getWorkOutputFilePath(context, fileName); + LOGGER.info("createInputStream: path = " + path); + final FSDataInputStream fsDataInputStream = path.getFileSystem(context.getConfiguration()).open(path); + return new BufferedInputStream(fsDataInputStream); + } private static class BytesCountingOutputStream extends OutputStream { diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java index d3345d1e9..115b3e8c8 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java @@ -76,6 +76,9 @@ public static class QLConfig { @Parameter private String shapefileURL; + @Parameter(defaultValue = "false") + private boolean wmsEnabled; + public String getImageType() { return imageType; } @@ -155,5 +158,10 @@ public void setLegendEnabled(boolean legendEnabled) { public void setOverlayURL(String overlayURL) { this.overlayURL = overlayURL; } + + public boolean isWmsEnabled() { + return wmsEnabled; + } + } } diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java index 222f186a5..f91675510 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java @@ -21,6 +21,7 @@ import com.bc.calvalus.processing.JobConfigNames; import com.bc.calvalus.processing.ProcessorAdapter; import com.bc.calvalus.processing.ProcessorFactory; +import com.bc.calvalus.processing.analysis.GeoServer; import com.bc.calvalus.processing.analysis.QLMapper; import com.bc.calvalus.processing.analysis.Quicklooks; import com.bc.calvalus.processing.hadoop.ProgressSplitProgressMonitor; @@ -39,6 +40,7 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.text.DateFormat; import java.util.*; import java.util.logging.Level; @@ -167,22 +169,48 @@ private void writeQuicklooks(Mapper.Context context, Configuration jobConfig, St List qlConfigList = getValidQlConfigs(jobConfig); for (Quicklooks.QLConfig qlConfig : qlConfigList) { - String imageFileName; + String imageBaseName; // if (context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REGEX) != null // && context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REPLACEMENT) != null) { -// imageFileName = L2FormattingMapper.getProductName(context.getConfiguration(), productName); +// imageBaseName = L2FormattingMapper.getProductName(context.getConfiguration(), productName); // } else { - imageFileName = productName; + imageBaseName = productName; // } if (qlConfigList.size() > 1) { - imageFileName = imageFileName + "_" + qlConfig.getBandName(); + imageBaseName = imageBaseName + "_" + qlConfig.getBandName(); } try { - QLMapper.createQuicklook(targetProduct, imageFileName, context, qlConfig); + QLMapper.createQuicklook(targetProduct, imageBaseName, context, qlConfig); } catch (Exception e) { String msg = String.format("Could not create quicklook image '%s'.", qlConfig.getBandName()); LOG.log(Level.WARNING, msg, e); } + + final String qlUploadHandler = context.getConfiguration().get(JobConfigNames.CALVALUS_QUICKLOOK_UPLOAD_HANDLER); + if (qlConfig.isWmsEnabled()) { + if (qlUploadHandler == null || qlUploadHandler.isEmpty()) { + LOG.warning("No quicklook upload handler configured"); + } + else if (qlUploadHandler.equalsIgnoreCase("geoserver")) { + if ("geotiff".equalsIgnoreCase(qlConfig.getImageType()) || ("cog".equalsIgnoreCase(qlConfig.getImageType()))) { + // upload geoTiff to GeoServer + try { + LOG.info(String.format("Quicklook upload handler: %s", qlUploadHandler)); + GeoServer geoserver = new GeoServer(context); + String imageFilename = QLMapper.getImageFileName(imageBaseName, qlConfig); + InputStream inputStream = QLMapper.createInputStream(context, imageFilename); + geoserver.uploadImage(inputStream, imageBaseName); + } catch (Exception e) { + String msg = String.format("Could not upload quicklook image '%s' to GeoServer.", qlConfig.getBandName()); + LOG.log(Level.WARNING, msg, e); + } + } else { + LOG.warning(String.format("Quicklook image format '%s' not supported for GeoServer upload. Please use am image format such as GeoTIFF or COG instead.", qlConfig.getImageType())); + } + } else { + LOG.warning(String.format("Unknown quicklook upload handler: %s", qlUploadHandler)); + } + } } LOG.info("Finished creating quicklooks."); } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java index 2e7ff5b1d..8d88dbe63 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java @@ -64,23 +64,17 @@ public Production createProduction(ProductionRequest productionRequest) throws P // todo - if autoStaging=true, create sequential workflow and add staging job String stagingDir = productionRequest.getStagingDirectory(productionId); - //boolean autoStaging = productionRequest.isAutoStaging(); //TODO - boolean autoStaging = false; + boolean autoStaging = productionRequest.isAutoStaging(); + String quicklookOutputDir = getOutputPath(productionRequest, productionId, ""); return new Production(productionId, productionName, - "", + quicklookOutputDir, stagingDir, autoStaging, productionRequest, workflowItem); } - // TODO, at the moment no staging implemented - @Override - protected Staging createUnsubmittedStaging(Production production) { - throw new UnsupportedOperationException("Staging currently not implemented for quick look generation."); - } - WorkflowItem createWorkflowItem(String productionId, String productionName, ProductionRequest productionRequest) throws ProductionException { From a662754f4410e10ca31088d72d7aab6dc59ee8d5 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Mon, 6 Sep 2021 17:14:28 +0100 Subject: [PATCH 2/3] Add quicklook support to the calvalus portal --- HowToDeploy.txt | 10 + .../portal/client/CalvalusPortal.java | 27 + .../portal/client/OrderL2ProductionView.java | 13 + .../portal/client/OrderQLProductionView.java | 195 ++++++ .../calvalus/portal/client/PortalContext.java | 3 + .../client/QuicklookParametersForm.java | 648 ++++++++++++++++++ .../portal/server/BackendServiceImpl.java | 24 + .../server/ColorPalettePersistence.java | 111 +++ .../portal/shared/BackendService.java | 20 + .../portal/shared/BackendServiceAsync.java | 4 + .../portal/shared/DtoColorPalette.java | 99 +++ .../com/bc/calvalus/portal/client/style.css | 62 ++ .../bc/calvalus/portal/CalvalusPortal.gwt.xml | 2 + .../client/QuicklookParametersForm.ui.xml | 401 +++++++++++ .../portal/server/color-palettes.properties | 26 + .../main/webapp/config/calvalus.properties | 11 + 16 files changed, 1656 insertions(+) create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java create mode 100644 calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml create mode 100644 calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties diff --git a/HowToDeploy.txt b/HowToDeploy.txt index b41c6087b..91368b5c0 100644 --- a/HowToDeploy.txt +++ b/HowToDeploy.txt @@ -221,5 +221,15 @@ calvalus.ql.upload.password = geoserver # GeoServer workspace to use. Mandatory. Example: calvalus.ql.upload.workspace = calmar +=========================================================================== +GeoServer - Calvalus portal support +----------------------------------- +Once GeoServer is enabled in the Calvalus production system, it is also possible to configure a +quicklooks WMS checkbox in the portal user interface. To enable this WMS checkbox please set the +following configuration option, this is an example: + +# Quicklooks WMS checkbox - access control and visual configuration +calvalus.portal.ql.wms = calvalus bc bg coastcolour calwps + diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java index 48560b5c3..8fe46b277 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java @@ -11,6 +11,7 @@ import com.bc.calvalus.portal.shared.ContextRetrievalServiceAsync; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; import com.bc.calvalus.portal.shared.DtoCalvalusConfig; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; import com.bc.calvalus.portal.shared.DtoProductSet; import com.bc.calvalus.portal.shared.DtoProduction; @@ -60,6 +61,7 @@ public class CalvalusPortal implements EntryPoint, PortalContext { "vicariousCalibrationView", "matchupComparisonView", "l2ToL3ComparisonView", + "qlView", "regionsView", "requestsView", "bundlesView", @@ -74,6 +76,7 @@ public class CalvalusPortal implements EntryPoint, PortalContext { // Data provided by various external services private ListDataProvider regions; private DtoProductSet[] productSets; + private DtoColorPalette[] colorPalettes; private DtoProcessorDescriptor[] systemProcessors; private DtoProcessorDescriptor[] userProcessors; private DtoProcessorDescriptor[] allUserProcessors; @@ -128,6 +131,7 @@ public void onModuleLoad() { Runnable runnable = new Runnable() { public void run() { backendService.loadRegions(NO_FILTER, new InitRegionsCallback()); + backendService.loadColorPalettes(NO_FILTER, new InitColorPaletteSetsCallback()); backendService.getProductSets(NO_FILTER, new InitProductSetsCallback()); final BundleFilter systemFilter = new BundleFilter(); @@ -182,6 +186,11 @@ public DtoProductSet[] getProductSets() { return productSets; } + @Override + public DtoColorPalette[] getColorPalettes() { + return colorPalettes; + } + @Override public DtoProcessorDescriptor[] getProcessors(String filter) { if (filter.equals(BundleFilter.PROVIDER_SYSTEM)) { @@ -285,6 +294,8 @@ private PortalView createViewOf(String name) { return new OrderMACProductionView(this); case "l2ToL3ComparisonView": return new OrderL2toL3ProductionView(this); + case "qlView": + return new OrderQLProductionView(this); case "regionsView": return new ManageRegionsView(this); case "bundlesView": @@ -482,6 +493,22 @@ public void onFailure(Throwable caught) { } } + private class InitColorPaletteSetsCallback implements AsyncCallback { + + @Override + public void onSuccess(DtoColorPalette[] dtoColorPalettes) { + CalvalusPortal.this.colorPalettes = dtoColorPalettes; + maybeInitFrontend(); + } + + @Override + public void onFailure(Throwable caught) { + caught.printStackTrace(System.err); + Dialog.error("Server-side Error", caught.getMessage()); + CalvalusPortal.this.colorPalettes = new DtoColorPalette[0]; + } + } + private class InitProcessorsCallback implements AsyncCallback { private final String filter; diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java index e1bb9f9d8..49562c9d1 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java @@ -18,6 +18,7 @@ import com.bc.calvalus.portal.shared.DtoInputSelection; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; +import com.bc.calvalus.portal.shared.DtoProcessorVariable; import com.google.gwt.core.client.Scheduler; import com.google.gwt.user.client.rpc.AsyncCallback; import com.google.gwt.user.client.ui.Anchor; @@ -44,6 +45,7 @@ public class OrderL2ProductionView extends OrderProductionView { private ProductsFromCatalogueForm productsFromCatalogueForm; private L2ConfigForm l2ConfigForm; private OutputParametersForm outputParametersForm; + private QuicklookParametersForm quicklookParametersForm; private Widget widget; public OrderL2ProductionView(PortalContext portalContext) { @@ -77,6 +79,7 @@ public void onClearSelectionClick() { } outputParametersForm = new OutputParametersForm(portalContext); + quicklookParametersForm = new QuicklookParametersForm(portalContext); l2ConfigForm.setProductSet(productSetSelectionForm.getSelectedProductSet()); handleProcessorChanged(); @@ -89,6 +92,7 @@ public void onClearSelectionClick() { } panel.add(l2ConfigForm); panel.add(outputParametersForm); + panel.add(quicklookParametersForm); Anchor l2Help = new Anchor("Show Help"); l2Help.getElement().getStyle().setProperty("textDecoration", "none"); l2Help.addStyleName("anchor"); @@ -168,6 +172,7 @@ protected HashMap getProductionParameters() { } parameters.putAll(l2ConfigForm.getValueMap()); parameters.putAll(outputParametersForm.getValueMap()); + parameters.putAll(quicklookParametersForm.getValueMap()); return parameters; } @@ -185,6 +190,7 @@ public void setProductionParameters(Map parameters) { } l2ConfigForm.setValues(parameters); outputParametersForm.setValues(parameters); + quicklookParametersForm.setValues(parameters); } private class InputSelectionCallback implements AsyncCallback { @@ -216,6 +222,13 @@ private void handleProcessorChanged() { add("BigGeoTiff", outputFormats); } outputParametersForm.setAvailableOutputFormats(outputFormats.toArray(new String[0])); + + List processorVariables = new ArrayList<>(); + DtoProcessorVariable[] dtoProcessorVariables = processorDescriptor.getProcessorVariables(); + for (DtoProcessorVariable dtoProcessorVariable : dtoProcessorVariables) { + processorVariables.add(dtoProcessorVariable.getName()); + } + quicklookParametersForm.setBandNames(processorVariables.toArray(new String[0])); } } diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java new file mode 100644 index 000000000..19f1c1ed6 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2013 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.portal.client; + +import com.bc.calvalus.portal.shared.DtoInputSelection; +import com.google.gwt.core.client.Scheduler; +import com.google.gwt.user.client.rpc.AsyncCallback; +import com.google.gwt.user.client.ui.Anchor; +import com.google.gwt.user.client.ui.VerticalPanel; +import com.google.gwt.user.client.ui.Widget; + +import java.util.HashMap; +import java.util.Map; + +/** + * Demo view that lets users submit a new L2 production. + * + * @author Norman + */ +public class OrderQLProductionView extends OrderProductionView { + + public static final String ID = OrderQLProductionView.class.getName(); + + private ProductSetSelectionForm productSetSelectionForm; + private ProductSetFilterForm productSetFilterForm; + private ProductsFromCatalogueForm productsFromCatalogueForm; + private OutputParametersForm outputParametersForm; + private QuicklookParametersForm quicklookParametersForm; + private Widget widget; + + public OrderQLProductionView(PortalContext portalContext) { + super(portalContext); + + productSetSelectionForm = new ProductSetSelectionForm(getPortal()); + productSetFilterForm = new ProductSetFilterForm(portalContext); + productSetFilterForm.setProductSet(productSetSelectionForm.getSelectedProductSet()); + + if (getPortal().withPortalFeature(INPUT_FILES_PANEL)) { + productsFromCatalogueForm = new ProductsFromCatalogueForm(getPortal()); + productsFromCatalogueForm.addInputSelectionHandler(new ProductsFromCatalogueForm.InputSelectionHandler() { + @Override + public AsyncCallback getInputSelectionChangedCallback() { + return new InputSelectionCallback(); + } + + @Override + public void onClearSelectionClick() { + productsFromCatalogueForm.removeSelections(); + } + }); + } + + outputParametersForm = new OutputParametersForm(portalContext); + outputParametersForm.showFormatSelectionPanel(false); + outputParametersForm.setAvailableOutputFormats("Image"); + + quicklookParametersForm = new QuicklookParametersForm(portalContext); + quicklookParametersForm.setBandNames(); + + VerticalPanel panel = new VerticalPanel(); + panel.setWidth("100%"); + panel.add(productSetSelectionForm); + panel.add(productSetFilterForm); + if (getPortal().withPortalFeature(INPUT_FILES_PANEL)){ + panel.add(productsFromCatalogueForm); + } + panel.add(outputParametersForm); + panel.add(quicklookParametersForm); + Anchor l2Help = new Anchor("Show Help"); + l2Help.getElement().getStyle().setProperty("textDecoration", "none"); + l2Help.addStyleName("anchor"); + panel.add(l2Help); + HelpSystem.addClickHandler(l2Help, "l2Processing"); + //panel.add(new HTML("
")); + panel.add(createOrderPanel()); + + this.widget = panel; + } + + @Override + public Widget asWidget() { + return widget; + } + + @Override + public String getViewId() { + return ID; + } + + @Override + public String getTitle() { + return "Quicklook generation"; + } + + @Override + protected String getProductionType() { + return "QL"; + } + + @Override + public void onShowing() { + // make sure #triggerResize is called after the new view is shown + Scheduler.get().scheduleFinally(() -> { + // See http://code.google.com/p/gwt-google-apis/issues/detail?id=127 + productSetFilterForm.getRegionMap().getMapWidget().triggerResize(); + }); + } + + @Override + protected boolean validateForm() { + try { + productSetSelectionForm.validateForm(); + productSetFilterForm.validateForm(); + if (productsFromCatalogueForm != null) { + productsFromCatalogueForm.validateForm(productSetSelectionForm.getSelectedProductSet().getName()); + } + outputParametersForm.validateForm(); + + + if (!getPortal().withPortalFeature("unlimitedJobSize")) { + try { + final int numDaysValue = Integer.parseInt(productSetFilterForm.numDays.getValue()); + if (numDaysValue > 365 + 366) { + throw new ValidationException(productSetFilterForm.numDays, "time range larger than allowed"); + } + } catch (NumberFormatException e) { + // ignore + } + } + return true; + } catch (ValidationException e) { + e.handle(); + return false; + } + } + + @Override + protected HashMap getProductionParameters() { + HashMap parameters = new HashMap<>(); + parameters.putAll(productSetSelectionForm.getValueMap()); + parameters.putAll(productSetFilterForm.getValueMap()); + if (productsFromCatalogueForm != null) { + parameters.putAll(productsFromCatalogueForm.getValueMap()); + } + parameters.putAll(outputParametersForm.getValueMap()); + parameters.putAll(quicklookParametersForm.getValueMap()); + return parameters; + } + + @Override + public boolean isRestoringRequestPossible() { + return true; + } + + @Override + public void setProductionParameters(Map parameters) { + productSetSelectionForm.setValues(parameters); + productSetFilterForm.setValues(parameters); + if (productsFromCatalogueForm != null) { + productsFromCatalogueForm.setValues(parameters); + } + outputParametersForm.setValues(parameters); + quicklookParametersForm.setValues(parameters); + } + + private class InputSelectionCallback implements AsyncCallback { + + @Override + public void onSuccess(DtoInputSelection inputSelection) { + Map inputSelectionMap = UIUtils.parseParametersFromContext(inputSelection); + productsFromCatalogueForm.setValues(inputSelectionMap); + productSetSelectionForm.setValues(inputSelectionMap); + productSetFilterForm.setValues(inputSelectionMap); + } + + @Override + public void onFailure(Throwable caught) { + Dialog.error("Error in retrieving input selection", caught.getMessage()); + } + } +} \ No newline at end of file diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java index 9cdd74dbd..5ec3eb96e 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java @@ -5,6 +5,7 @@ import com.bc.calvalus.portal.shared.BackendServiceAsync; import com.bc.calvalus.portal.shared.ContextRetrievalServiceAsync; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; import com.bc.calvalus.portal.shared.DtoProductSet; import com.bc.calvalus.portal.shared.DtoProduction; @@ -27,6 +28,8 @@ public interface PortalContext { // make this return ListDataProvider DtoProductSet[] getProductSets(); + DtoColorPalette[] getColorPalettes(); + // make this return ListDataProvider DtoProcessorDescriptor[] getProcessors(String filter); DtoAggregatorDescriptor[] getAggregators(String filter); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java new file mode 100644 index 000000000..84a5be820 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -0,0 +1,648 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.portal.client; + +import com.bc.calvalus.portal.shared.DtoColorPalette; +import com.google.gwt.core.client.GWT; +import com.google.gwt.dom.client.Element; +import com.google.gwt.dom.client.Style; +import com.google.gwt.event.dom.client.ChangeEvent; +import com.google.gwt.event.dom.client.ChangeHandler; +import com.google.gwt.event.logical.shared.ValueChangeEvent; +import com.google.gwt.event.logical.shared.ValueChangeHandler; +import com.google.gwt.uibinder.client.UiBinder; +import com.google.gwt.uibinder.client.UiField; +import com.google.gwt.user.client.DOM; +import com.google.gwt.user.client.ui.CheckBox; +import com.google.gwt.user.client.ui.Composite; +import com.google.gwt.user.client.ui.DisclosurePanel; +import com.google.gwt.user.client.ui.HTMLPanel; +import com.google.gwt.user.client.ui.IntegerBox; +import com.google.gwt.user.client.ui.Label; +import com.google.gwt.user.client.ui.ListBox; +import com.google.gwt.user.client.ui.RadioButton; +import com.google.gwt.user.client.ui.TextBox; +import com.google.gwt.user.client.ui.Widget; +import com.google.gwt.xml.client.Document; +import com.google.gwt.xml.client.Node; +import com.google.gwt.xml.client.NodeList; +import com.google.gwt.xml.client.XMLParser; + +import java.util.HashMap; +import java.util.Map; + +/** + * This form is used regarding visualisations settings. + * + * @author Declan + */ +public class QuicklookParametersForm extends Composite { + + interface TheUiBinder extends UiBinder { + } + + private static TheUiBinder uiBinder = GWT.create(TheUiBinder.class); + + private static int instanceCounter = 0; + private int radioGroupId = 0; + + private static final String CUSTOM_BAND = ""; + + private static final String BANDNAME_LISTBOX_ROW = "bandNameListBoxRow"; + + private final PortalContext portal; + + @UiField + HTMLPanel singleBandPanelBasics; + @UiField + HTMLPanel singleBandPanelAdvanced; + @UiField + HTMLPanel multiBandPanelBasics; + @UiField + HTMLPanel multiBandPanelAdvanced; + @UiField + HTMLPanel wmsPanel; + @UiField + DisclosurePanel advancedOptionsPanel; + + @UiField + RadioButton quicklookNone; + @UiField + RadioButton quicklookSingleBand; + @UiField + RadioButton quicklookMultiBand; + + @UiField + ListBox bandNameListBox; + @UiField + Label bandNameRowLabel; + @UiField + TextBox bandName; + @UiField + ListBox colorPalette; + + @UiField + TextBox rgbaExpressionsRedBand; + @UiField + TextBox rgbaExpressionsGreenBand; + @UiField + TextBox rgbaExpressionsBlueBand; + @UiField + TextBox rgbaExpressionsAlphaBand; + @UiField + TextBox rgbaMinSamples; + @UiField + TextBox rgbaMaxSamples; + + @UiField + ListBox imageType; + @UiField + TextBox overlayURL; + @UiField + TextBox maskOverlays; + @UiField + IntegerBox subSamplingX; + @UiField + IntegerBox subSamplingY; + @UiField + TextBox backgroundColor; + @UiField + CheckBox legendEnabled; + + @UiField + CheckBox wmsEnabled; + + private DtoColorPalette[] availableColorPalettes = null; + private Boolean pageLoaded = false; + private String[] availableBandNames = null; + private Boolean wmsAvailable; + private final String DEFAULT_COLOR_PALETTE = "spectrum"; + + public QuicklookParametersForm(PortalContext portalContext) { + this.portal = portalContext; + initWidget(uiBinder.createAndBindUi(this)); + + instanceCounter++; + radioGroupId = instanceCounter; + + //give radio group a unique name for each instance + quicklookNone.setName("quicklookSelection" + radioGroupId); + quicklookSingleBand.setName("quicklookSelection" + radioGroupId); + quicklookMultiBand.setName("quicklookSelection" + radioGroupId); + + quicklookNone.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + if (event.getValue()) + quicklookNoneChangeHandler(); + } + }); + + quicklookSingleBand.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + if (event.getValue()) + quicklookSingleBandChangeHandler(); + } + }); + + quicklookMultiBand.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + if (event.getValue()) + quicklookMultiBandChangeHandler(); + } + }); + + bandName.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + bandNameChangeHandler(); + } + }); + + bandNameListBox.addChangeHandler(new ChangeHandler() { + @Override + public void onChange(ChangeEvent event) { + bandNameListBoxChangeHandler(); + } + }); + + setAvailableImageTypes(); + setColorPalettes(); + + if (portalContext.withPortalFeature("ql.wms")) { + wmsAvailable = true; + } else { + wmsAvailable = false; + wmsPanel.setVisible(false); + } + } + + @Override + protected void onLoad() { + //give HTML element a unique id for each instance + Element element = DOM.getElementById(BANDNAME_LISTBOX_ROW); + element.setId(BANDNAME_LISTBOX_ROW + radioGroupId); + + this.pageLoaded = true; + quicklookNone.setValue(true, true); + buildBandNameListBox(); + } + + private void bandNameChangeHandler() { + // band name field changed, reset list box index and call it's handler + int itemCount = bandNameListBox.getItemCount(); + String bandNameValue = bandName.getValue(); + bandNameListBox.setSelectedIndex(0); + for (int i = 0; i < itemCount; i++) { + String listBand = bandNameListBox.getValue(i); + if (listBand != null && listBand.equals(bandNameValue)) { + bandNameListBox.setSelectedIndex(i); + break; + } + } + bandNameListBoxChangeHandler(); + } + + private void bandNameListBoxChangeHandler() { + if (!pageLoaded) + return; + + int selectedIndex = bandNameListBox.getSelectedIndex(); + Element element = DOM.getElementById(BANDNAME_LISTBOX_ROW + radioGroupId); + if (!quicklookSingleBand.getValue() || selectedIndex < 0) { + // single band not selected OR no values in listbox + // enable bandName textbox, make drop down list invisible + bandNameRowLabel.setText("Band name:"); + bandName.setEnabled(true); + if (element != null) + element.getStyle().setDisplay(Style.Display.NONE); + } else { + // single band is selected AND values in listbox + // make drop down list visible + if (element != null) + element.getStyle().setDisplay(Style.Display.TABLE_ROW); + bandNameRowLabel.setText(""); + if (selectedIndex == 0) { + // custom band name option selected + // enable bandName textbox + bandName.setEnabled(true); + } else if (selectedIndex > 0) { + // band name (sourced from processor) selected + // disable bandName textbox + bandName.setValue(bandNameListBox.getValue(selectedIndex)); + bandName.setEnabled(false); + } + } + } + + private void quicklookNoneChangeHandler() { + singleBandPanelBasics.setVisible(false); + singleBandPanelAdvanced.setVisible(false); + multiBandPanelBasics.setVisible(false); + multiBandPanelAdvanced.setVisible(false); + advancedOptionsPanel.setVisible(false); + if (wmsAvailable) { + wmsPanel.setVisible(false); + } + } + + private void quicklookSingleBandChangeHandler() { + singleBandPanelBasics.setVisible(true); + singleBandPanelAdvanced.setVisible(true); + multiBandPanelBasics.setVisible(false); + multiBandPanelAdvanced.setVisible(false); + advancedOptionsPanel.setVisible(true); + if (wmsAvailable) { + wmsPanel.setVisible(true); + } + bandNameChangeHandler(); + } + + private void quicklookMultiBandChangeHandler() { + singleBandPanelBasics.setVisible(false); + singleBandPanelAdvanced.setVisible(false); + multiBandPanelBasics.setVisible(true); + multiBandPanelAdvanced.setVisible(true); + advancedOptionsPanel.setVisible(true); + if (wmsAvailable) { + wmsPanel.setVisible(true); + } + } + + private void setAvailableImageTypes() { + String[] imageNames = {"png", "jpeg", "tiff", "geotiff", "cog"}; + int selectedIndex = imageType.getSelectedIndex(); + imageType.clear(); + for (String imageName : imageNames) { + imageType.addItem(imageName); + } + if (selectedIndex >= 0 && selectedIndex < imageNames.length) { + imageType.setSelectedIndex(selectedIndex); + } else { + imageType.setSelectedIndex(0); + } + } + + private void setColorPalettes() { + this.availableColorPalettes = portal.getColorPalettes(); + colorPalette.clear(); + int index = 0; + for (int i = 0; i < this.availableColorPalettes.length; i++) { + DtoColorPalette dtoColorPalette = this.availableColorPalettes[i]; + String qualifiedName = dtoColorPalette.getQualifiedName(); + colorPalette.addItem(qualifiedName, dtoColorPalette.getCpdURL()); + if (qualifiedName.equals(DEFAULT_COLOR_PALETTE)) { + index = i; + } + } + colorPalette.setSelectedIndex(index); + } + + public void setBandNames(String... bandNames) { + this.availableBandNames = bandNames; + buildBandNameListBox(); + } + + private void buildBandNameListBox() { + bandNameListBox.clear(); + if (!pageLoaded) + return; + + if (availableBandNames != null && availableBandNames.length > 0) { + bandNameListBox.addItem(CUSTOM_BAND); + bandNameListBox.setSelectedIndex(0); + for (String bandName : this.availableBandNames) { + bandNameListBox.addItem(bandName); + } + } + bandNameChangeHandler(); + } + + public Map getValueMap() { + Map parameters = new HashMap(); + if (quicklookNone.getValue()) { + return parameters; + } + + String indentXML = " "; + + // imageType + String imageTypeXML = indentXML + "" + imageType.getSelectedValue() + "\n"; + + // overlayURL + String overlayURLValue = overlayURL.getValue(); + String overlayURLXML = ""; + if (overlayURLValue != null && !overlayURLValue.isEmpty()) { + overlayURLXML = indentXML + "" + overlayURLValue + "\n"; + } + + // maskOverlays + String maskOverlaysValue = maskOverlays.getValue(); + String maskOverlaysXML = ""; + if (maskOverlaysValue != null && !maskOverlaysValue.isEmpty()) { + maskOverlaysXML = indentXML + "" + maskOverlaysValue + "\n"; + } + + // subSamplingX + Integer subSamplingXValue = subSamplingX.getValue(); + String subSamplingXXML = ""; + if (subSamplingXValue != null && subSamplingXValue >= 1) { + subSamplingXXML = indentXML + "" + subSamplingXValue + "\n"; + } + + // subSamplingY + Integer subSamplingYValue = subSamplingY.getValue(); + String subSamplingYXML = ""; + if (subSamplingYValue != null && subSamplingYValue >= 1) { + subSamplingYXML = indentXML + "" + subSamplingYValue + "\n"; + } + + // backgroundColor + String backgroundColorValue = backgroundColor.getValue(); + String backgroundColorXML = ""; + if (backgroundColorValue != null && !backgroundColorValue.isEmpty()) { + backgroundColorXML = indentXML + "" + backgroundColorValue + "\n"; + } + + // legendEnabled + Boolean legendEnabledValue = legendEnabled.getValue(); + String legendEnabledXML = ""; + if (legendEnabledValue) { + legendEnabledXML = indentXML + "true\n"; + } + + // wmsEnabled + Boolean wmsEnabledValue = wmsEnabled.getValue(); + String wmsEnabledXML = ""; + if (wmsEnabledValue) { + wmsEnabledXML = indentXML + "true\n"; + } + + String quicklookParameters = ""; + if (quicklookSingleBand.getValue()) { + + // bandName + String bandNameXML = ""; + String bandNameValue = bandName.getValue(); + if (bandNameValue != null && !bandNameValue.isEmpty()) { + bandNameXML = indentXML + "" + bandNameValue + "\n"; + } + + // color palette + String cpdURLXML = ""; + String colorPaletteValue = colorPalette.getSelectedValue(); + if (colorPaletteValue != null && !colorPaletteValue.isEmpty()) { + cpdURLXML = indentXML + "" + colorPaletteValue + "\n"; + } + + quicklookParameters = "\n" + + " \n" + + " \n" + + bandNameXML + + cpdURLXML + + imageTypeXML + + overlayURLXML + + maskOverlaysXML + + subSamplingXXML + + subSamplingYXML + + backgroundColorXML + + legendEnabledXML + + wmsEnabledXML + + " \n" + + " \n" + + ""; + } else if (quicklookMultiBand.getValue()) { + + // RGBAExpressions + String rgbaExpressionsXML = ""; + String rgbaExpressionsRedBandValue = rgbaExpressionsRedBand.getValue(); + String rgbaExpressionsGreenBandValue = rgbaExpressionsGreenBand.getValue(); + String rgbaExpressionsBlueBandValue = rgbaExpressionsBlueBand.getValue(); + String rgbaExpressionsAlphaBandValue = rgbaExpressionsAlphaBand.getValue(); + + if (rgbaExpressionsRedBandValue != null && !rgbaExpressionsRedBandValue.isEmpty() && + rgbaExpressionsGreenBandValue != null && !rgbaExpressionsGreenBandValue.isEmpty() && + rgbaExpressionsBlueBandValue != null && !rgbaExpressionsBlueBandValue.isEmpty()) { + + rgbaExpressionsXML = indentXML + "" + + rgbaExpressionsRedBandValue + "," + + rgbaExpressionsGreenBandValue + "," + + rgbaExpressionsBlueBandValue; + + if (rgbaExpressionsAlphaBandValue != null && !rgbaExpressionsAlphaBandValue.isEmpty()) + rgbaExpressionsXML = rgbaExpressionsXML + "," + rgbaExpressionsAlphaBandValue; + + rgbaExpressionsXML = rgbaExpressionsXML + "\n"; + } + + // rgbaMinSamples + String rgbaMinSamplesXML = ""; + String rgbaMinSamplesValue = rgbaMinSamples.getValue(); + if (rgbaMinSamplesValue != null && !rgbaMinSamplesValue.isEmpty()) { + rgbaMinSamplesXML = indentXML + "" + rgbaMinSamplesValue + "\n"; + } + + // rgbaMaxSamples + String rgbaMaxSamplesXML = ""; + String rgbaMaxSamplesValue = rgbaMaxSamples.getValue(); + if (rgbaMaxSamplesValue != null && !rgbaMaxSamplesValue.isEmpty()) { + rgbaMaxSamplesXML = indentXML + "" + rgbaMaxSamplesValue + "\n"; + } + + quicklookParameters = "\n" + + " \n" + + " \n" + + rgbaExpressionsXML + + rgbaMinSamplesXML + + rgbaMaxSamplesXML + + imageTypeXML + + overlayURLXML + + maskOverlaysXML + + subSamplingXXML + + subSamplingYXML + + backgroundColorXML + + legendEnabledXML + + wmsEnabledXML + + " \n" + + " \n" + + ""; + } + + if (!quicklookParameters.isEmpty()) { + parameters.put("quicklooks", "true"); + parameters.put("calvalus.ql.parameters", quicklookParameters); + } + + return parameters; + } + + public void setValues(Map parameters) { + String quicklookParametersValue = parameters.get("calvalus.ql.parameters"); + if (quicklookParametersValue == null || quicklookParametersValue.isEmpty()) { + + bandName.setValue(null); + colorPalette.setSelectedIndex(0); + + rgbaExpressionsRedBand.setValue(null); + rgbaExpressionsGreenBand.setValue(null); + rgbaExpressionsBlueBand.setValue(null); + rgbaExpressionsAlphaBand.setValue(null); + rgbaMinSamples.setValue(null); + rgbaMaxSamples.setValue(null); + + imageType.setSelectedIndex(0); + overlayURL.setValue(null); + maskOverlays.setValue(null); + subSamplingX.setValue(null); + maskOverlays.setValue(null); + legendEnabled.setValue(false); + + wmsEnabled.setValue(false); + + // set no quicklook radio button in GUI + quicklookNone.setValue(true, true); + } else { + Document dom = XMLParser.parse(quicklookParametersValue); + + // bandName + String bandNameValue = getTagValue(dom, "bandName"); + if (bandNameValue != null) { + bandName.setValue(bandNameValue); + + // As bandName has a value, assume this is a single band + // Therefore, set single band radio button in GUI + quicklookSingleBand.setValue(true, true); + } + + // color palette + String cpdURLValue = getTagValue(dom, "cpdURL"); + if (cpdURLValue == null || this.availableColorPalettes == null) { + colorPalette.setSelectedIndex(0); + } else { + for (int i = 0; i < this.availableColorPalettes.length; i++) { + if (this.availableColorPalettes[i].getCpdURL().equals(cpdURLValue)) { + colorPalette.setSelectedIndex(i); + break; + } + } + } + + // RGBAExpressions + String RGBAExpressionsValue = getTagValue(dom, "RGBAExpressions"); + if (RGBAExpressionsValue != null) { + String[] tokens = RGBAExpressionsValue.split(","); + int numTokens = tokens.length; + rgbaExpressionsRedBand.setValue(numTokens > 0 ? tokens[0] : null); + rgbaExpressionsGreenBand.setValue(numTokens > 1 ? tokens[1] : null); + rgbaExpressionsBlueBand.setValue(numTokens > 2 ? tokens[2] : null); + rgbaExpressionsAlphaBand.setValue(numTokens > 3 ? tokens[3] : null); + + // As RGBAExpressions has a value, assume this is a multi band, + // Therefore, set multi band radio button in GUI + quicklookMultiBand.setValue(true, true); + } else { + rgbaExpressionsRedBand.setValue(null); + rgbaExpressionsGreenBand.setValue(null); + rgbaExpressionsBlueBand.setValue(null); + rgbaExpressionsAlphaBand.setValue(null); + } + + // rgbaMinSamples + String rgbaMinSamplesValue = getTagValue(dom, "rgbaMinSamples"); + rgbaMinSamples.setValue(rgbaMinSamplesValue); + + // rgbaMaxSamples + String rgbaMaxSamplesValue = getTagValue(dom, "rgbaMaxSamples"); + rgbaMaxSamples.setValue(rgbaMaxSamplesValue); + + // imageType + String imageTypeValue = getTagValue(dom, "imageType"); + if (imageTypeValue == null) + imageType.setSelectedIndex(0); + else if (imageTypeValue.equalsIgnoreCase("png")) + imageType.setSelectedIndex(0); + else if (imageTypeValue.equalsIgnoreCase("jpeg")) + imageType.setSelectedIndex(1); + else if (imageTypeValue.equalsIgnoreCase("tiff")) + imageType.setSelectedIndex(2); + else if (imageTypeValue.equalsIgnoreCase("geotiff")) + imageType.setSelectedIndex(3); + else if (imageTypeValue.equalsIgnoreCase("cog")) + imageType.setSelectedIndex(4); + + // overlayURL + String overlayURLValue = getTagValue(dom, "overlayURL"); + overlayURL.setValue(overlayURLValue); + + // maskOverlays + String maskOverlaysValue = getTagValue(dom, "maskOverlays"); + maskOverlays.setValue(maskOverlaysValue); + + // subSamplingX + String subSamplingXValue = getTagValue(dom, "subSamplingX"); + try { + Integer subSamplingXIntValue = Integer.valueOf(subSamplingXValue); + subSamplingX.setValue(subSamplingXIntValue); + } catch (NumberFormatException e) { + subSamplingX.setValue(null); + } + + // subSamplingY + String subSamplingYValue = getTagValue(dom, "subSamplingY"); + try { + Integer subSamplingYIntValue = Integer.valueOf(subSamplingYValue); + subSamplingY.setValue(subSamplingYIntValue); + } catch (NumberFormatException e) { + subSamplingY.setValue(null); + } + + // backgroundColor + String backgroundColorValue = getTagValue(dom, "backgroundColor"); + backgroundColor.setValue(backgroundColorValue); + + // legendEnabled + String legendEnabledValue = getTagValue(dom, "legendEnabled"); + if (legendEnabledValue != null && legendEnabledValue.equalsIgnoreCase("true")) + legendEnabled.setValue(true); + else + legendEnabled.setValue(false); + + // wmsEnabled + String wmsEnabledValue = getTagValue(dom, "wmsEnabled"); + if (wmsEnabledValue != null && wmsEnabledValue.equalsIgnoreCase("true")) + wmsEnabled.setValue(true); + else + wmsEnabled.setValue(false); + } + } + + private String getTagValue(Document dom, String tagName) { + NodeList nodeList = dom.getElementsByTagName(tagName); + if (nodeList != null && nodeList.getLength() > 0) { + Node node = nodeList.item(0); + if (node != null && node.hasChildNodes()) { + Node firstChild = node.getFirstChild(); + if (firstChild != null) { + return (firstChild.getNodeValue()); + } + } + } + return null; + } +} \ No newline at end of file diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java index f99ff1b6b..9f7dd8e8a 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java @@ -29,6 +29,7 @@ import com.bc.calvalus.portal.shared.BackendServiceException; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; import com.bc.calvalus.portal.shared.DtoCalvalusConfig; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.bc.calvalus.portal.shared.DtoMaskDescriptor; import com.bc.calvalus.portal.shared.DtoParameterDescriptor; import com.bc.calvalus.portal.shared.DtoProcessState; @@ -247,6 +248,29 @@ public void storeRegions(DtoRegion[] regions) throws BackendServiceException { } } + @Override + public DtoColorPalette[] loadColorPalettes(String filter) throws BackendServiceException { + ColorPalettePersistence colorPalettePersistence = new ColorPalettePersistence(getUserName(), ProductionServiceConfig.getUserAppDataDir()); + try { + DtoColorPalette[] dtoColorPalettes = colorPalettePersistence.loadColorPalettes(); + LOG.fine("loadColorPalettes returns " + dtoColorPalettes.length); + return dtoColorPalettes; + } catch (IOException e) { + log(e.getMessage(), e); + throw new BackendServiceException("Failed to load color palettes: " + e.getMessage(), e); + } + } + + @Override + public void storeColorPalettes(DtoColorPalette[] colorPalettes) throws BackendServiceException { + ColorPalettePersistence colorPalettePersistence = new ColorPalettePersistence(getUserName(), ProductionServiceConfig.getUserAppDataDir()); + try { + colorPalettePersistence.storeColorPalettes(colorPalettes); + } catch (IOException e) { + throw new BackendServiceException("Failed to store color palettes: " + e.getMessage(), e); + } + } + @Override public DtoProductSet[] getProductSets(String filter) throws BackendServiceException { if (filter.contains("dummy")) { diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java new file mode 100644 index 000000000..664c2973a --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java @@ -0,0 +1,111 @@ +package com.bc.calvalus.portal.server; + +import com.bc.calvalus.portal.shared.DtoColorPalette; + +import java.io.*; +import java.util.*; + +/** + * A location that stores color palettes. + * + * @author Declan + */ +public class ColorPalettePersistence { + + private final String userName; + private final File userAppDataDir; + + public ColorPalettePersistence(String userName, File userAppDataDir) { + this.userName = userName; + this.userAppDataDir = userAppDataDir; + } + + public DtoColorPalette[] loadColorPalettes() throws IOException { + Properties colorPaletteProperties = loadDefaultColorPalettes(); + Properties userColorPaletteProperties = loadUserColorPalettes(); + colorPaletteProperties.putAll(userColorPaletteProperties); + + ArrayList colorPalettes = new ArrayList(); + Set colorPaletteNames = colorPaletteProperties.stringPropertyNames(); + for (String colorPaletteFullName : colorPaletteNames) { + String[] split = colorPaletteFullName.split("\\."); + String colorPaletteName = split[split.length - 1]; + String[] colorPalettePath = Arrays.copyOf(split, split.length - 1); + String cpdURL = colorPaletteProperties.getProperty(colorPaletteFullName); + DtoColorPalette colorPalette = new DtoColorPalette(colorPaletteName, colorPalettePath, cpdURL); + colorPalettes.add(colorPalette); + } + + Collections.sort(colorPalettes, new Comparator() { + @Override + public int compare(DtoColorPalette o1, DtoColorPalette o2) { + return o1.getQualifiedName().compareToIgnoreCase(o2.getQualifiedName()); + } + }); + return colorPalettes.toArray(new DtoColorPalette[colorPalettes.size()]); + } + + public void storeColorPalettes(DtoColorPalette[] colorPalettes) throws IOException { + Properties userColorPalettes = getUserColorPalettes(colorPalettes); + File file = getColorPaletteFile(getUserName()); + file.getParentFile().mkdirs(); + FileWriter fileWriter = new FileWriter(file); + try { + userColorPalettes.store(fileWriter, "Calvalus color palettes for user " + getUserName()); + } finally { + fileWriter.close(); + } + } + + private File getColorPaletteFile(String user) { + return new File(userAppDataDir, user + "-color-palettes.properties"); + } + + private String getUserName() { + return userName; + } + + private Properties getUserColorPalettes(DtoColorPalette[] colorPalettes) { + Properties userColorPalettes = new Properties(); + for (DtoColorPalette colorPalette : colorPalettes) { + if (colorPalette.isUserColorPalette()) { + String fullName = colorPalette.getQualifiedName(); + System.out.println("storing color palette " + fullName + " = " + colorPalette.getCpdURL()); + userColorPalettes.put(fullName, colorPalette.getCpdURL()); + } + } + return userColorPalettes; + } + + private Properties loadDefaultColorPalettes() throws IOException { + InputStream stream = getClass().getResourceAsStream("color-palettes.properties"); + Properties systemColorPalettes = loadColorPalettes(new BufferedReader(new InputStreamReader(stream))); + + File additionalSystemColorPaletteFile = getColorPaletteFile("SYSTEM"); + if (additionalSystemColorPaletteFile.exists()) { + Properties additionalSystemColorPalettes = loadColorPalettes(new FileReader(additionalSystemColorPaletteFile)); + systemColorPalettes.putAll(additionalSystemColorPalettes); + } + return systemColorPalettes; + } + + private Properties loadUserColorPalettes() throws IOException { + File userColorPaletteFile = getColorPaletteFile(getUserName()); + if (userColorPaletteFile.exists()) { + return loadColorPalettes(new FileReader(userColorPaletteFile)); + } else { + return new Properties(); + } + } + + private Properties loadColorPalettes(Reader stream) throws IOException { + Properties colorPalettes = new Properties(); + try { + colorPalettes.load(stream); + } finally { + stream.close(); + } + return colorPalettes; + } + +} \ No newline at end of file diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java index d2024b5d0..f1525f35e 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java @@ -32,6 +32,26 @@ public interface BackendService extends RemoteService { */ void storeRegions(DtoRegion[] regions) throws BackendServiceException; + /** + * Gets all known color palettes. + * + * @param filter A filter expression (not yet used). + * + * @return The array of color palettes. + * + * @throws BackendServiceException If a server error occurred. + */ + DtoColorPalette[] loadColorPalettes(String filter) throws BackendServiceException; + + /** + * Persists the provided color palettes. + * + * @param colorPalettes The color palettes to persist. + * + * @throws BackendServiceException If a server error occurred. + */ + void storeColorPalettes(DtoColorPalette[] colorPalettes) throws BackendServiceException; + /** * Gets all known product sets. * diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java index 165fb7198..1bad84fcd 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java @@ -13,6 +13,10 @@ public interface BackendServiceAsync { void storeRegions(DtoRegion[] regions, AsyncCallback callback); + void loadColorPalettes(String filter, AsyncCallback callback); + + void storeColorPalettes(DtoColorPalette[] colorPalettes, AsyncCallback callback); + void listUserFiles(String dir, AsyncCallback callback); void removeUserFile(String path, AsyncCallback callback); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java new file mode 100644 index 000000000..295958239 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.portal.shared; + +import com.google.gwt.user.client.rpc.IsSerializable; + +/** + * @author Declan + */ +public class DtoColorPalette implements IsSerializable { + + private String name; + private String[] path; + private String cpdURL; + + /** + * No-arg constructor as required by {@link IsSerializable}. Don't use directly. + */ + public DtoColorPalette() { + } + + public DtoColorPalette(String name, String[] path, String cpdURL) { + if (name == null) { + throw new NullPointerException("name"); + } + if (path == null) { + throw new NullPointerException("path"); + } + if (path == null) { + throw new NullPointerException("cpdURL"); + } + this.name = name; + this.path = path; + this.cpdURL = cpdURL; + } + + public String getName() { + return name; + } + + public String[] getPath() { + return path; + } + + public String getCpdURL() { + return cpdURL; + } + + public boolean isUserColorPalette() { + return path != null && path.length > 0 && "user".equals(path[0]); + } + + public String getQualifiedName() { + StringBuilder sb = new StringBuilder(); + for (String pathElement : path) { + sb.append(pathElement); + sb.append("."); + } + sb.append(name); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DtoColorPalette that = (DtoColorPalette) o; + + if (!name.equals(that.name)) { + return false; + } + if (!path.equals(that.path)) { + return false; + } + if (!path.equals(that.cpdURL)) { + return false; + } + return true; + } +} \ No newline at end of file diff --git a/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css b/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css index c65a29fe3..1736e4094 100644 --- a/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css +++ b/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css @@ -118,6 +118,68 @@ font-size: 14px; } +/************************** +* Quicklook Panel styling * +***************************/ +.quicklookPanel { + margin: 0px 15px 50px 15px; + width: 62em; +} + +.quicklookRadioPanel { + padding: 5px 0px 5px 0px; +} + +.quicklookRadioButton { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; + color: black; + padding-right: 10px; +} + +.quicklookParametersPanel { + display: flex; +} + +.quicklookParametersPanelIndent { + display: flex; + border-left: 3px solid white; + padding: 4px 0px 4px 8px; + margin-left: 6px; +} + +.quicklookParametersTable { + width: 100%; +} + +.quicklookParametersTableHeader { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; + font-weight: normal; + text-align: left; + background-color: #cddaee; + padding: 2px; + border-radius: 5px; +} + +.quicklookParametersTableRowPadding { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; + width: 185px; + padding-top: 8px; +} + +.quicklookInputLabelCell { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; + width: 185px; +} + +.quicklookInputCell { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; +} + /********************** * Input files styling * ***********************/ diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml index e7a6bf344..3ad07324a 100644 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml @@ -17,6 +17,8 @@ + + diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml new file mode 100644 index 000000000..d0342966d --- /dev/null +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -0,0 +1,401 @@ + + + + + + + + + Quicklook + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Single-band configuration
+
+ Band name: + + +
+ + + +
    + + The name of a band which will be used for the image. + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Multi-band configuration
+
+ Red band expression: + + +
+ Green band expression: + + +
+ Blue band expression: + + +
+ Alpha band expression: + + +
    + + Add a list of 3 or 4 band names from which an RGB or RGBA colourised image will be + generated. + +
+ + + + + Advanced options + + + + + + + + + + + + + + + + + + + +
Single-band configuration - + more options +
+
+ Colour table: + + +
    + + The colour table which will be applied to the chosen band. + +
+ + + + + + + + + + + + + + + + + + + + + + + +
Multi-band configuration - + more options +
+
+ Minimum samples: + + +
+ Maximum samples: + + +
    + + Add a list of 3 or 4 values that set minimal and maximal values for the displayed + samples. + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
General options
+
+ Image type: + + +
    + + The type of the generated image. + +
   
+ Background Colour: + + +
    + + The background colour of the generated image. + +
   
+ Overlay URL: + + +
    + + A URL to a file containing a image that will be rendered above the image from the + product. + +
   
+ Mask overlays: + + +
    + + Defines which product mask should be drawn on top of + the actual band or RGB image. The product must contain + the specified masks. + +
   
+ Sub sampling (X): + + +
+ Sub sampling (Y): + + +
    + + Sub sampling can be used to reduce the size of the resulting image. + The default size of the image is identical to the size of the product. + +
   
+ Legend + + +
    + + When set, an image legend will be drawn into the bottom-right corner of the image. + +
+ + + +
+ + + + + + + + + + + + + + + + + + + +
Web Map Service
+
+ WMS + + +
    + + When set, the generated quicklook image is uploaded to a WMS server. + +
+ + + + + + +
+ + + + + \ No newline at end of file diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties new file mode 100644 index 000000000..69dde0080 --- /dev/null +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties @@ -0,0 +1,26 @@ +5_colors = hdfs://master00:9000/calvalus/auxiliary/cpd/5_colors.cpd +7_colors = hdfs://master00:9000/calvalus/auxiliary/cpd/7_colors.cpd +cc_chl = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_chl.cpd +cc_general = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_general.cpd +cc_iop_quality = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_iop_quality.cpd +cc_tsm = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_tsm.cpd +cc_yellowsubstance = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_yellowsubstance.cpd +cc_z90 = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_z90.cpd +CHL_SeaWiFS = hdfs://master00:9000/calvalus/auxiliary/cpd/CHL_SeaWiFS.cpd +cubehelix_cycle = hdfs://master00:9000/calvalus/auxiliary/cpd/cubehelix_cycle.cpd +gradient_8_colors = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_8_colors.cpd +gradient_blue = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_blue.cpd +gradient_green = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_green.cpd +gradient_red = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_red.cpd +gradient_red_white_blue = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_red_white_blue.cpd +great_circle = hdfs://master00:9000/calvalus/auxiliary/cpd/great_circle.cpd +JET = hdfs://master00:9000/calvalus/auxiliary/cpd/JET.cpd +meris_algal = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_algal.cpd +meris_case2 = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_case2.cpd +meris_cloud = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_cloud.cpd +meris_pressure = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_pressure.cpd +meris_veg_index = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_veg_index.cpd +meris_wind_direction = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_wind_direction.cpd +spectrum = hdfs://master00:9000/calvalus/auxiliary/cpd/spectrum.cpd +spectrum_large = hdfs://master00:9000/calvalus/auxiliary/cpd/spectrum_large.cpd +terrain = hdfs://master00:9000/calvalus/auxiliary/cpd/terrain.cpd diff --git a/calvalus-portal/src/main/webapp/config/calvalus.properties b/calvalus-portal/src/main/webapp/config/calvalus.properties index d077f9020..4c2cd4ab1 100644 --- a/calvalus-portal/src/main/webapp/config/calvalus.properties +++ b/calvalus-portal/src/main/webapp/config/calvalus.properties @@ -71,6 +71,7 @@ calvalus.portal.bootstrappingView = calvalus bc calvalus.portal.vicariousCalibrationView = calvalus bc calvalus.portal.matchupComparisonView = calvalus bc calvalus.portal.l2ToL3ComparisonView = calvalus bc +calvalus.portal.qlView = calvalus bc bg coastcolour calwps calvalus.portal.regionsView = calvalus bc bg coastcolour calwps calvalus.portal.bundlesView = calvalus bc calwps calvalus.portal.masksView = calvalus bc bg @@ -81,6 +82,16 @@ calvalus.portal.catalogue = bc calvalus.portal.unlimitedJobSize = calvalus bc calvalus.portal.inputFilesPanel = +# Quicklook WMS checkbox - access control and visual configuration +calvalus.portal.ql.wms = calvalus bc bg coastcolour calwps + +# Configuration for publishing quicklook GeoTiff images to GeoServer +calvalus.ql.upload.handler = geoserver +calvalus.ql.upload.URL = http://localhost:8080/geoserver/rest +calvalus.ql.upload.username = admin +calvalus.ql.upload.password = geoserver +calvalus.ql.upload.workspace = calmar + # optional production queue to submit jobs to when using the portal # default is default calvalus.hadoop.mapreduce.job.queuename = default From f511652f319bbb5e6d2ad3670c940a988a802bdd Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Sun, 23 Jan 2022 21:34:08 +0000 Subject: [PATCH 3/3] Updates for quicklook processing of COG image * Fix subsampling bug * Use GDAL MEM driver instead of GTiff driver to prepare image * Use LZW compression * Update GDAL to version 3.4.0 --- calvalus-processing/pom.xml | 2 +- .../processing/analysis/QLMapper.java | 63 +++++++++---------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/calvalus-processing/pom.xml b/calvalus-processing/pom.xml index ed2fb3fe1..c1ea5b1ed 100644 --- a/calvalus-processing/pom.xml +++ b/calvalus-processing/pom.xml @@ -230,7 +230,7 @@ org.gdal gdal - 3.3.0 + 3.4.0 diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java index 4f7b6878b..5787e15fb 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java @@ -66,7 +66,7 @@ import java.io.OutputStream; import java.util.HashMap; import java.util.Map; -//import java.util.Vector; +import java.util.Vector; import java.util.logging.Logger; /** @@ -145,49 +145,42 @@ public static void createQuicklook(Product product, String imageBaseName, Mapper product = GPF.createProduct("Reproject", reprojParams, product); quicklookImage = new QuicklookGenerator(context, product, config).createImage(); if (quicklookImage != null) { - final int width = product.getSceneRasterWidth(); - final int height = product.getSceneRasterHeight(); + final int datasetWidth = product.getSceneRasterWidth(); + final int datasetHeight = product.getSceneRasterHeight(); final GeoCoding geoCoding = product.getSceneGeoCoding(); final PixelPos posA = new org.esa.snap.core.datamodel.PixelPos(0, 0); final GeoPos geoPosA = geoCoding.getGeoPos(posA, null); - final PixelPos posB = new org.esa.snap.core.datamodel.PixelPos(width, height); + final PixelPos posB = new org.esa.snap.core.datamodel.PixelPos(datasetWidth, datasetHeight); final GeoPos geoPosB = geoCoding.getGeoPos(posB, null); if (isCloudOptimizedGeoTIFF(config)) { // use GDAL Java bindings API to create a Cloud Optimized GeoTIFF (COG) gdal.AllRegister(); - Driver driverGeoTiff = gdal.GetDriverByName("GTiff"); - if (driverGeoTiff == null) { - throw new RuntimeException("Could not load GDAL GTiff driver required for Cloud Optimized GeoTIFF (COG)"); + Driver driverMemory = gdal.GetDriverByName("MEM"); + if (driverMemory == null) { + throw new RuntimeException("Could not load GDAL MEM driver required for Cloud Optimized GeoTIFF (COG)"); } Driver driverCOG = gdal.GetDriverByName("COG"); if (driverCOG == null) { throw new RuntimeException("Could not load GDAL COG driver required for Cloud Optimized GeoTIFF (COG)"); } Raster quicklookRaster = quicklookImage.getData(); - int numberOfRasterBands = quicklookRaster.getNumBands(); + int quicklookImageWidth = quicklookRaster.getWidth(); + int quicklookImageHeight = quicklookRaster.getHeight(); + int quicklookImageNumBands = quicklookRaster.getNumBands(); String tmpImageFileName = getTmpImageFileName(imageBaseName, config); - /* - // Add GDAL specific control parameters for the GTiff driver if required in future. Example: - final Vector geoTiffCreationOptions = new Vector<>(4); - imageCreationOptions.add("COMPRESS=JPEG"); - imageCreationOptions.add("TILED=FALSE"); - imageCreationOptions.add("SPARSE_OK=FALSE"); - imageCreationOptions.add("PROFILE=GeoTIFF"); - Dataset geoTiffDataset = driverGeoTiff.Create(tmpImageFileName, width, height, numberOfRasterBands, gdalconst.GDT_Byte, geoTiffCreationOptions); - */ - Dataset geoTiffDataset = driverGeoTiff.Create(tmpImageFileName, width, height, numberOfRasterBands, gdalconst.GDT_Byte); + Dataset memoryDataset = driverMemory.Create(tmpImageFileName, quicklookImageWidth, quicklookImageHeight, quicklookImageNumBands, gdalconst.GDT_Byte); // add SRS to the GDAL dataset object SpatialReference srs = new SpatialReference(); srs.ImportFromEPSG(4326); - geoTiffDataset.SetSpatialRef(srs); + memoryDataset.SetSpatialRef(srs); // add Geotransform (north up image) to the GDAL dataset object // assumes longitude is between -90 and 90, and latitude between -180 and 180 - double pixelSizeX = Math.abs(geoPosA.getLon() - geoPosB.getLon()) / (double) width; - double pixelSizeY = Math.abs(geoPosA.getLat() - geoPosB.getLat()) / (double) height; + double pixelSizeX = Math.abs(geoPosA.getLon() - geoPosB.getLon()) / (double) quicklookImageWidth; + double pixelSizeY = Math.abs(geoPosA.getLat() - geoPosB.getLat()) / (double) quicklookImageHeight; double[] geotransform = new double[6]; geotransform[0] = geoPosA.getLon(); // longitude value for the top left pixel of the raster geotransform[1] = pixelSizeX; // pixel width @@ -195,26 +188,30 @@ public static void createQuicklook(Product product, String imageBaseName, Mapper geotransform[3] = geoPosA.getLat(); // latitude value for the top left pixel of the raster geotransform[4] = 0; // 0 geotransform[5] = -pixelSizeY; // pixel height (negative value) - geoTiffDataset.SetGeoTransform(geotransform); + memoryDataset.SetGeoTransform(geotransform); // add image data to the GDAL dataset object, for each image band - int[] band = new int[(height * width)]; - for (int i = 0; i < numberOfRasterBands; i++) { - quicklookRaster.getSamples(0, 0, width, height, i, band); - Band outBand = geoTiffDataset.GetRasterBand(i + 1); - outBand.WriteRaster(0, 0, width, height, gdalconst.GDT_Int32, band); + int[] band = new int[(quicklookImageHeight * quicklookImageWidth)]; + for (int i = 0; i < quicklookImageNumBands; i++) { + quicklookRaster.getSamples(0, 0, quicklookImageWidth, quicklookImageHeight, i, band); + Band outBand = memoryDataset.GetRasterBand(i + 1); + outBand.WriteRaster(0, 0, quicklookImageWidth, quicklookImageHeight, gdalconst.GDT_Int32, band); outBand.FlushCache(); } - geoTiffDataset.FlushCache(); + memoryDataset.FlushCache(); - // create and copy COG image file using the GeoTIFF dataset - Dataset cogDataset = driverCOG.CreateCopy(imageFileName, geoTiffDataset); + // create and copy COG image file using the in-memory dataset + // add GDAL specific format parameters for the COG + final Vector formatParameters = new Vector<>(1); + //formatParameters.add("COMPRESS=NONE"); + formatParameters.add("COMPRESS=LZW"); + Dataset cogDataset = driverCOG.CreateCopy(imageFileName, memoryDataset, formatParameters); - // must do this cleanup now, to get all header and data to disk. - // Otherwise the COG image becomes corrupt + // flush header and data to disk and then delete the objects + // from past testing the COG image became corrupt without this now cogDataset.FlushCache(); cogDataset.delete(); - geoTiffDataset.delete(); + memoryDataset.delete(); // copy local Cloud Optimized GeoTIFF file to the HDFS filesystem FileSystem localFilesystem = FileSystem.getLocal(context.getConfiguration());