V3.0.0

.gitlab-ci.yml

@@ -13,18 +13,18 @@ include:
       pipeline_enable_sonarqube: "false"
       lint_yamllint_enable: "false"
 
-security:filesystem:scan 🛡️:
-  rules:
-    - when: always
+# security:filesystem:scan 🛡️:
+  # rules:
+    # - when: always
 
-check_container-image_scanning:
-  stage: security
-  image: alpine:latest
-  script:
-    - CRITICAL_COUNT=$(grep -o '<tr class="severity-CRITICAL">' scanning-report.html | wc -l || echo 0) 
-    - echo "Number of critical vulnerabilities:${CRITICAL_COUNT}"
-    - if [ "$CRITICAL_COUNT" -gt 0 ]; then echo "Critical vulnerabilities detected"; exit 1; fi
-  needs:
-    - job: security:filesystem:scan 🛡️
-      artifacts: true
-  allow_failure: false  
+# check_container-image_scanning:
+  # stage: security
+  # image: alpine:latest
+  # script:
+    # - CRITICAL_COUNT=$(grep -o '<tr class="severity-CRITICAL">' scanning-report.html | wc -l || echo 0) 
+    # - echo "Number of critical vulnerabilities:${CRITICAL_COUNT}"
+    # - if [ "$CRITICAL_COUNT" -gt 0 ]; then echo "Critical vulnerabilities detected"; exit 1; fi
+  # needs:
+    # - job: security:filesystem:scan 🛡️
+      # artifacts: true
+  # allow_failure: false  

README.md

@@ -14,6 +14,7 @@ Vocabularies and format changes are managed on [Argo Vocabs Task Team - AVTT Git
 **With each release (from 2.9.2) you will find :**
 
 - **file_checker_exec-[version].jar** which consolidates both application's compiled code and all its dependencies into a single executable file.
+- file_checker_spec folder containing all specifications and reference tables.
 - source code
 
 ## Run Argo NetCDF file format checker
@@ -114,7 +115,7 @@ output files will be generated in `./demo/outputs`.
 
 ### Run File checker using Python and an API
 
-In folder /file-checker-python you will find a python wrapper and an API to facilitate the use of Argo FileChecker. See python.README for more informations.
+In folder /file-checker-python you will find a python wrapper and an API to facilitate the use of Argo FileChecker. See [python.README](./python.README.md) for more informations.
 
 ### Test data
 

file_checker_exec/.gitignore

@@ -1,3 +1,4 @@
 /bin/
 /target/
-ValidateSubmit_LOG
+ValidateSubmit_LOG
+*.log

file_checker_exec/.settings/org.eclipse.core.resources.prefs

@@ -1,6 +1,7 @@
 eclipse.preferences.version=1
 encoding//src/main/java=UTF-8
 encoding//src/main/resources=UTF-8
+encoding//src/main/resources/file_checker_spec=UTF-8
 encoding//src/test/java=UTF-8
 encoding//src/test/resources=UTF-8
 encoding/<project>=UTF-8

file_checker_exec/pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>fr.ifremer</groupId>
 	<artifactId>file_checker_exec</artifactId>
-	<version>2.9.5</version>
+	<version>3.0.0</version>
 
 	<name>Argo NetCDF file format checker</name>
 
@@ -13,6 +13,9 @@
 		<dep.junit.version>5.11.4</dep.junit.version>
 		<maven.compiler.source>1.8</maven.compiler.source>
     	<maven.compiler.target>1.8</maven.compiler.target>
+
+    	<cz.cvut.kbss.jb4jsonld.version>0.15.4</cz.cvut.kbss.jb4jsonld.version>
+        <com.fasterxml.jackson.version>2.18.3</com.fasterxml.jackson.version>
 	</properties>
 
 	<dependencies>
@@ -81,6 +84,17 @@
 		</dependency>
 
 
+		<dependency>
+		    <groupId>com.io-informatics.oss</groupId>
+		    <artifactId>jackson-jsonld</artifactId>
+		    <version>0.1.1</version>
+		</dependency>
+		<dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jakarta-jsonp</artifactId>
+            <version>${com.fasterxml.jackson.version}</version>
+        </dependency>
+
 	</dependencies>
 
 	<!--Distribution-->
@@ -118,14 +132,25 @@
 		<sourceDirectory>src/main/java</sourceDirectory>
 		<testSourceDirectory>src/test/java</testSourceDirectory>
 		<resources>
+
+
 			<resource>
 				<filtering>true</filtering>
 				<directory>src/main/resources</directory>
 				<excludes>
 					<exclude>**/*.java</exclude>
 					<exclude>**/log4j2*</exclude>
+					<exclude>file_checker_spec/**</exclude>					
 				</excludes>
 			</resource>
+
+			<!--included spec dir for online mode : no filtering -->
+			<resource>
+		      <directory>src/main/resources/file_checker_spec</directory>
+		      <filtering>false</filtering>
+		      <targetPath>file_checker_spec</targetPath>
+    		</resource>
+
 		</resources>
 		<testResources>
 			<testResource>

file_checker_exec/src/main/java/fr/coriolis/checker/config/Options.java

@@ -32,7 +32,6 @@
  * <li>-format-only-pre3.1: Perform format-only checks for versions
  * pre-3.1.</li>
  * <li>-data-check-all: Perform data checks for all files.</li>
- * <li>-battery-check: Enable battery variable checks.</li>
  * <li>-psal-stats: Compute PSAL statistics for index files.</li>
  * <li>-list-file &lt;file&gt;: Specify a file containing a list of input
  * files.</li>
@@ -54,7 +53,6 @@ public class Options {
 
 	private static Options instance;
 
-	private final boolean doBatteryChecks; // ..check metadate battery vars - default: no
 	private final boolean doNameCheck; // ..check file name - default: yes
 	private final boolean doNulls; // ..check for nulls in strings - default: no
 	private final boolean doFormatOnly; // ..true: format-only; false: format and data checks
@@ -71,18 +69,22 @@ public class Options {
 	private final String outDirName;
 	private final String inDirName;
 
+	private final boolean useOnlineNVS; // to read NVS tables from internet instead of the ones providing in the spec
+										// dir (internal or external)
+	private final boolean useInternalSpecs; // to use specs file now included in the .jar file instead of specifying an
+											// external file_chec_spec dir
+
 	// ..standard i/o shortcuts
 	static PrintStream stdout = new PrintStream(System.out);
 	static PrintStream stderr = new PrintStream(System.err);
 	private static final Logger log = LogManager.getLogger("Options");
 
 	// ====== CONSTRUCTOR ======
-	private Options(boolean doBatteryChecks, boolean doNameCheck, boolean doNulls, boolean doFormatOnly,
-			boolean doFormatOnlyPre31, boolean doPsalStats, boolean version, boolean help, boolean doXml,
-			String listFile, List<String> inFileList, String dacName, String specDirName, String outDirName,
-			String inDirName) {
+	private Options(boolean doNameCheck, boolean doNulls, boolean doFormatOnly, boolean doFormatOnlyPre31,
+			boolean doPsalStats, boolean version, boolean help, boolean doXml, String listFile, List<String> inFileList,
+			String dacName, String specDirName, String outDirName, String inDirName, boolean useOnlineNVS,
+			boolean useInternalSpecs) {
 		super();
-		this.doBatteryChecks = doBatteryChecks;
 		this.doNameCheck = doNameCheck;
 		this.doNulls = doNulls;
 		this.doFormatOnly = doFormatOnly;
@@ -98,7 +100,9 @@ private Options(boolean doBatteryChecks, boolean doNameCheck, boolean doNulls, b
 		this.outDirName = outDirName;
 		this.inDirName = inDirName;
 
-		log.debug("doBatteryChecks = {}", doBatteryChecks);
+		this.useOnlineNVS = useOnlineNVS;
+		this.useInternalSpecs = useInternalSpecs;
+
 		log.debug("doFormatOnly = {}", doFormatOnly);
 		log.debug("doFormatOnlyPre31 = {}", doFormatOnlyPre31);
 		log.debug("doNameCheck = {}", doNameCheck);
@@ -110,24 +114,37 @@ private Options(boolean doBatteryChecks, boolean doNameCheck, boolean doNulls, b
 		log.debug("outDirName = '{}'", outDirName);
 		log.debug("inDirName = '{}'", inDirName);
 		log.debug("number of inFileList = " + (inFileList == null ? "null" : inFileList.size()));
+		log.debug("useOnlineNVS = {}", useOnlineNVS);
+		log.debug("useInternalSpecs = {}", useInternalSpecs);
 	}
 
 	/**
-	 * Retrieves the singleton instance of {@code Options}, parsing command-line
-	 * arguments with {@code extractOptionsFromArgs} if instance is null .
 	 * 
-	 * @param args : The command-line arguments passed to the application.
 	 * @return The singleton instance of {@code Options}.
 	 */
-	public static synchronized Options getInstance(String args[]) throws IllegalArgumentException {
+	public static synchronized Options getInstance() throws IllegalArgumentException {
 
 		if (instance == null) {
 
-			instance = extractOptionsFromArgs(args);
+			throw new IllegalStateException("Options not initialized");
 		}
 		return instance;
 	}
 
+	/**
+	 * initiate the singleton of {@code Options}, parsing command-line arguments
+	 * with {@code extractOptionsFromArgs} if instance is null .
+	 * 
+	 * @param args : The command-line arguments passed to the application.
+	 */
+	public static void init(String args[]) {
+		if (instance != null) {
+			return;
+		} else {
+			instance = extractOptionsFromArgs(args);
+		}
+	}
+
 	/**
 	 * Parse command-line arguments. Exit application if bad arguments are provided.
 	 * 
@@ -139,7 +156,6 @@ private static Options extractOptionsFromArgs(String[] args) throws IllegalArgum
 		// Default values :
 		String listFile = null;
 		List<String> inFileList = null; // ..list of input files//..list file name
-		boolean doBatteryChecks = false; // ..check metadate battery vars - default: no
 		boolean doNameCheck = true; // ..check file name - default: yes
 		boolean doNulls = false; // ..check for nulls in strings - default: no
 		boolean doFormatOnly = false; // ..true: format-only; false: format and data checks
@@ -150,6 +166,9 @@ private static Options extractOptionsFromArgs(String[] args) throws IllegalArgum
 		boolean help = false;
 		boolean doXml = true;
 
+		boolean useOnlineNVS = false;
+		boolean useInternalSpecs = false;
+
 		// loop trough the arguments provided and differentiate the option (start with
 		// "-") and the positional parameters.
 		int next = 0;
@@ -183,12 +202,15 @@ private static Options extractOptionsFromArgs(String[] args) throws IllegalArgum
 			case "-data-check-all":
 				doFormatOnlyPre31 = false;
 				break;
-			case "-battery-check":
-				doBatteryChecks = true;
-				break;
 			case "-psal-stats":
 				doPsalStats = true;
 				break;
+			case "-online-nvs":
+				useOnlineNVS = true;
+				break;
+			case "-internal-specs":
+				useInternalSpecs = true;
+				break;
 			case "-list-file":
 				if (++next < args.length) {
 					listFile = args[next];
@@ -197,6 +219,7 @@ private static Options extractOptionsFromArgs(String[] args) throws IllegalArgum
 					throw new IllegalArgumentException("Error: Missing argument after '-list-file'.");
 				}
 				break;
+
 			// ..obsolete arguments -- left in for backwards compatibility
 			case "-no-fresh":
 				log.error("Obsolete argument '-no-fresh' given. IGNORED");
@@ -212,22 +235,25 @@ private static Options extractOptionsFromArgs(String[] args) throws IllegalArgum
 		}
 
 		// .....parse the positional parameters.....
-		validateNumberOfPositionalArguments(args, next); // exit system if too few arguments
+		validateNumberOfPositionalArguments(args, next, useInternalSpecs); // exit system if too few arguments
 
 		String dacName = args[next++];
-		String specDirName = args[next++];
+		String specDirName = "";
+
+		if (!useInternalSpecs) {
+			specDirName = args[next++];
+		}
 		String outDirName = args[next++];
 		String inDirName = args[next++];
 		if (next < args.length) {
 			inFileList = new ArrayList<String>(args.length - next);
 			for (; next < args.length; next++) {
 				inFileList.add(args[next]);
-
 			}
 		}
 
-		return new Options(doBatteryChecks, doNameCheck, doNulls, doFormatOnly, doFormatOnlyPre31, doPsalStats, version,
-				help, doXml, listFile, inFileList, dacName, specDirName, outDirName, inDirName);
+		return new Options(doNameCheck, doNulls, doFormatOnly, doFormatOnlyPre31, doPsalStats, version, help, doXml,
+				listFile, inFileList, dacName, specDirName, outDirName, inDirName, useOnlineNVS, useInternalSpecs);
 
 	}
 
@@ -238,8 +264,10 @@ private static Options extractOptionsFromArgs(String[] args) throws IllegalArgum
 	 * @param args list of arguments
 	 * @param next indice of the next argument
 	 */
-	private static void validateNumberOfPositionalArguments(String[] args, int next) throws IllegalArgumentException {
-		if (args.length < (4 + next)) {
+	private static void validateNumberOfPositionalArguments(String[] args, int next, boolean useInternalSpecs)
+			throws IllegalArgumentException {
+		int minArgs = (useInternalSpecs ? 3 : 4) + next;
+		if (args.length < minArgs) {
 			log.error("too few arguments: " + args.length);
 			throw new IllegalArgumentException("Too few arguments provided.");
 		}
@@ -253,8 +281,8 @@ private static void validateNumberOfPositionalArguments(String[] args, int next)
 	 */
 	public void validateMandatoryArguments() {
 		checkDacName(dacName);
-		checkDirectory(inDirName);
-		checkDirectory(specDirName);
+		checkDirectory(inDirName, false);
+		checkDirectory(specDirName, useInternalSpecs);
 	}
 
 	/**
@@ -282,19 +310,15 @@ protected static void checkDacName(String dacName) {
 	 * 
 	 * @param directoryName
 	 */
-	protected static void checkDirectory(String directoryName) {
+	protected static void checkDirectory(String directoryName, boolean useOnlineTables) {
 		File dir = new File(directoryName);
-		if (!dir.isDirectory()) {
+		if (!useOnlineTables && !dir.isDirectory()) {
 			throw new IllegalArgumentException("ERROR: " + directoryName + " directory is not a directory");
 		}
 	}
 
 	// ===== GETTERS =====
 
-	public boolean isDoBatteryChecks() {
-		return doBatteryChecks;
-	}
-
 	public boolean isDoNameCheck() {
 		return doNameCheck;
 	}
@@ -351,4 +375,12 @@ public String getInDirName() {
 		return inDirName;
 	}
 
+	public boolean isUseOnlineNVS() {
+		return useOnlineNVS;
+	}
+
+	public boolean isUseInternalSpecs() {
+		return useInternalSpecs;
+	}
+
 }