Pull request #167: Support for authentication in user-provided remote artifacts (OAuth2.0, HTTP basic, HTTP header)

Merge in ITB/json-validator from development to master

* commit '0c64acc10af66345d2e7249f5c4aa2f988e45697':
  Support for authentication in user-provided remote artifacts (OAuth2.0, HTTP basic, HTTP header)

Author: costas80

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/validation/JSONValidator.java

@@ -52,7 +52,6 @@
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -251,14 +250,14 @@ private List<Message> validateAgainstSetOfSchemas(List<FileInfo> schemaFileInfos
         if (combinationApproach == ValidationArtifactCombinationApproach.ALL) {
             // All schema validations must result in success.
             for (FileInfo fileInfo: schemaFileInfos) {
-                aggregatedMessages.addAll(validateAgainstSchema(fileInfo.getFile()));
+                aggregatedMessages.addAll(validateAgainstSchema(fileInfo));
             }
         } else if (combinationApproach == ValidationArtifactCombinationApproach.ONE_OF) {
             // All schemas need to be validated but only one should validate successfully.
             int successCount = 0;
             int branchCounter = 1;
             for (FileInfo fileInfo: schemaFileInfos) {
-                var latestErrors = validateAgainstSchema(fileInfo.getFile());
+                var latestErrors = validateAgainstSchema(fileInfo);
                 if (latestErrors.isEmpty()) {
                     successCount += 1;
                 } else {
@@ -277,7 +276,7 @@ private List<Message> validateAgainstSetOfSchemas(List<FileInfo> schemaFileInfos
             // Any of the schemas should validate successfully.
             int branchCounter = 1;
             for (FileInfo fileInfo: schemaFileInfos) {
-                List<Message> latestErrors = validateAgainstSchema(fileInfo.getFile());
+                List<Message> latestErrors = validateAgainstSchema(fileInfo);
                 if (latestErrors.isEmpty()) {
                     aggregatedMessages.clear();
                     break;
@@ -312,12 +311,12 @@ private void addBranchErrors(List<Message> aggregatedMessages, List<Message> bra
      * The schema loading in this case extends what the official spec foresees, allowing to read definitions
      * from local files (and reuse schemas).
      *
-     * @param path The schema path.
+     * @param schemaInfo The schema information.
      * @return The parsed schema and JSON node reader to use.
      */
-    private Pair<Schema, NodeReader> readSchema(Path path) {
+    private Pair<Schema, NodeReader> readSchema(FileInfo schemaInfo) {
         try {
-            var jsonNode = objectMapper.readTree(path.toFile());
+            var jsonNode = objectMapper.readTree(schemaInfo.getFile());
             var jsonReader = getJsonReader();
             var registryConfig = SchemaRegistryConfig.builder()
                     .cacheRefs(false)
@@ -330,7 +329,7 @@ private Pair<Schema, NodeReader> readSchema(Path path) {
                     .schemaRegistryConfig(registryConfig)
                     .schemaCacheEnabled(false)
                     .nodeReader(jsonReader)
-                    .resourceLoaders(builder -> builder.add(new LocalSchemaResolver(specs.getDomainConfig(), localSchemaCache)))
+                    .resourceLoaders(builder -> builder.add(new LocalSchemaResolver(specs.getDomainConfig(), localSchemaCache, schemaInfo)))
                     .build();
             return Pair.of(registry.getSchema(jsonNode), jsonReader);
         } catch (IOException e) {
@@ -372,8 +371,8 @@ private Function<Error, String> getLocationMapper() {
      * @param schemaFile The schema file to use.
      * @return The resulting error messages.
      */
-    private List<Message> validateAgainstSchema(File schemaFile) {
-        var schemaInfo = readSchema(schemaFile.toPath());
+    private List<Message> validateAgainstSchema(FileInfo schemaFile) {
+        var schemaInfo = readSchema(schemaFile);
         var locationMapper = getLocationMapper();
         return schemaInfo.getLeft().validate(getContentNode(schemaInfo.getRight())).stream().map(error -> {
             String pathPrefix = null;

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/validation/LocalSchemaResolver.java

@@ -17,13 +17,16 @@
 
 import com.networknt.schema.AbsoluteIri;
 import com.networknt.schema.resource.InputStreamSource;
-import com.networknt.schema.resource.IriResourceLoader;
 import com.networknt.schema.resource.ResourceLoader;
+import com.networknt.schema.utils.AbsoluteIris;
 import eu.europa.ec.itb.json.DomainConfig;
+import eu.europa.ec.itb.validation.commons.FileInfo;
+import eu.europa.ec.itb.validation.commons.URLReader;
 import org.apache.commons.lang3.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.URI;
 import java.nio.file.Files;
 
 /**
@@ -35,16 +38,20 @@ public class LocalSchemaResolver implements ResourceLoader {
 
     private final DomainConfig domain;
     private final LocalSchemaCache localSchemaCache;
+    private final FileInfo schemaInfo;
+    private URLReader urlReader;
 
     /**
      * Constructor.
      *
      * @param domain The configuration domain.
      * @param localSchemaCache The cache of locally-defined schemas.
+     * @param schemaInfo The schema information.
      */
-    public LocalSchemaResolver(DomainConfig domain, LocalSchemaCache localSchemaCache) {
+    public LocalSchemaResolver(DomainConfig domain, LocalSchemaCache localSchemaCache, FileInfo schemaInfo) {
         this.domain = domain;
         this.localSchemaCache = localSchemaCache;
+        this.schemaInfo = schemaInfo;
     }
 
     /**
@@ -59,11 +66,23 @@ public InputStreamSource getResource(AbsoluteIri absoluteIri) {
         var schema = localSchemaCache.getSchemaForId(domain, idToCheck);
         if (schema.isEmpty()) {
             LOG.debug("Schema with URI {} not found locally. Looking up remotely.", absoluteIri);
-            return IriResourceLoader.getInstance().getResource(absoluteIri);
+            return () -> getUrlReader().stream(URI.create(AbsoluteIris.toUri(absoluteIri)), null, domain.getHttpVersion(), schemaInfo.getRequestDecorator()).stream();
         } else {
             LOG.debug("Schema with URI {} found locally.", absoluteIri);
             return () -> Files.newInputStream(schema.get());
         }
     }
 
+    /**
+     * Get the URL reader to use.
+     *
+     * @return The reader.
+     */
+    private URLReader getUrlReader() {
+        if (urlReader == null) {
+            urlReader = new URLReader();
+        }
+        return urlReader;
+    }
+
 }

jsonvalidator-web/src/main/java/eu/europa/ec/itb/json/web/UploadController.java

@@ -58,7 +58,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
 import java.util.*;
+import java.util.function.Consumer;
 
 import static eu.europa.ec.itb.validation.commons.web.Constants.*;
 
@@ -213,7 +215,7 @@ public UploadResult<Translations> handleUpload(@PathVariable("domain") String do
                     List<FileInfo> externalSchemas = new ArrayList<>();
                     boolean proceedToValidate = true;
                     try {
-                        externalSchemas = getExternalFiles(externalSchemaContentType, externalSchemaFiles, externalSchemaUri, externalSchemaString, config.getSchemaInfo(validationType), tempFolderForRequest, config.getHttpVersion());
+                        externalSchemas = getExternalFiles(externalSchemaContentType, externalSchemaFiles, externalSchemaUri, externalSchemaString, config.getSchemaInfo(validationType), tempFolderForRequest, config.getHttpVersion(), config);
                     } catch (IOException e) {
                         logger.error("Error while reading uploaded file [{}]", e.getMessage(), e);
                         result.setMessage(localisationHelper.localise("validator.label.exception.errorInUpload", e.getMessage()));
@@ -374,13 +376,15 @@ private ValidationArtifactCombinationApproach getSchemaCombinationApproach(Strin
      * @param schemaInfo The schema information from the domain.
      * @param parentFolder The temporary folder to use for file system storage.
      * @param httpVersion The HTTP version to use.
+     * @param domainConfig The domain configuration.
      * @return The list of user-provided artifacts.
      * @throws IOException If an IO error occurs.
      */
     private List<FileInfo> getExternalFiles(String[] externalContentType, MultipartFile[] externalFiles, String[] externalUri, String[] externalString,
-                                            ValidationArtifactInfo schemaInfo, File parentFolder, HttpClient.Version httpVersion) throws IOException {
+                                            ValidationArtifactInfo schemaInfo, File parentFolder, HttpClient.Version httpVersion, DomainConfig domainConfig) throws IOException {
         List<FileInfo> lis = new ArrayList<>();
         if (externalContentType != null) {
+            Consumer<HttpRequest.Builder> requestDecorator = fileManager.createRemoteFileRequestDecorator(domainConfig, schemaInfo);
             for (int i=0; i<externalContentType.length; i++) {
                 if (StringUtils.isNotBlank(externalContentType[i])) {
                     File inputFile;
@@ -396,10 +400,9 @@ private List<FileInfo> getExternalFiles(String[] externalContentType, MultipartF
                     if (externalString != null && externalString.length > i) {
                         currentExtString = externalString[i];
                     }
-
-                    inputFile = getInputFile(externalContentType[i], currentExtFile, currentExtUri, currentExtString, parentFolder, httpVersion);
+                    inputFile = getInputFile(externalContentType[i], currentExtFile, currentExtUri, currentExtString, parentFolder, httpVersion, requestDecorator);
                     if (inputFile != null) {
-                        lis.add(new FileInfo(inputFile));
+                        lis.add(new FileInfo(inputFile, null, null, requestDecorator));
                     }
                 }
             }
@@ -452,10 +455,11 @@ private boolean validateExternalFiles(List<FileInfo> lis, ValidationArtifactInfo
      * @param inputString The provided direct input string to load the content from.
      * @param parentFolder The temporary folder to use.
      * @param httpVersion The HTTP version to use.
+     * @param requestDecorator The request decorator to use.
      * @return The input content's file.
      * @throws IOException If an error occurs.
      */
-    private File getInputFile(String contentType, MultipartFile inputFile, String inputUri, String inputString, File parentFolder, HttpClient.Version httpVersion) throws IOException {
+    private File getInputFile(String contentType, MultipartFile inputFile, String inputUri, String inputString, File parentFolder, HttpClient.Version httpVersion, Consumer<HttpRequest.Builder> requestDecorator) throws IOException {
         File file = null;
         if (CONTENT_TYPE_FILE.equals(contentType)) {
             if (inputFile!=null && !inputFile.isEmpty()) {
@@ -465,7 +469,7 @@ private File getInputFile(String contentType, MultipartFile inputFile, String in
             }
         } else if (CONTENT_TYPE_URI.equals(contentType)) {
             if (StringUtils.isNotBlank(inputUri)) {
-                file = this.fileManager.getFileFromURL(parentFolder, inputUri, httpVersion);
+                file = this.fileManager.getFileFromURL(parentFolder, inputUri, null, null, null, null, null, null, httpVersion, requestDecorator).getFile();
             }
         } else if (CONTENT_TYPE_STRING.equals(contentType)) {
             if (StringUtils.isNotBlank(inputString)) {