From 8246000442a9acc129aee21d917ca91082d9cbcb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 25 Apr 2025 03:20:59 +0000 Subject: [PATCH 01/79] chore(deps): update jacksonversion to v2.19.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index cd6c189f..314b84dc 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,7 @@ repositories { } ext { - jacksonVersion = '2.18.3' + jacksonVersion = '2.19.0' } dependencies { From 5ee65fd6050d3eb3cab036d985eaf556f6bbe968 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:07:01 +0000 Subject: [PATCH 02/79] chore(deps): update dependency gradle to v8.14 --- gradle/wrapper/gradle-wrapper.jar | Bin 43705 -> 43764 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 4 ++-- gradlew.bat | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 9bbc975c742b298b441bfb90dbc124400a3751b9..1b33c55baabb587c669f562ae36f953de2481846 100644 GIT binary patch delta 642 zcmdmamFde>rVZJA^}0Q$xegf!xPEW^+5YDM%iT2bEgct9o+jH~+sJas#HZ=szO|** z=Pj=X_vx?W&DSwKck|WWn~hffsvnQ+42*W$b7b0$SCcOoZ`{W{^$^pk;4>8-A*-)$ z?n(Po`1$6Jn_u?t-L+tsPyZ2#X}8T6OS8pAU;kdgd+_Hw4z4TW0p9E!T+=f7-c&O% zFic^X{7^$?^Ho04eona9n#mGMxKhA=~8B%JN`M zMhm5wc-2v)$``sY$!Q`9xiU@DhI73ZxiGEKg>yIPs)NmWwMdF-ngLXpZSqV5ez36n zVkxF2rjrjWR+_xr6e6@_u@s~2uv{9vi*1pj2)BjFD+-%@&pRVP1f{O1glxTOp2-62Ph;v z`N1+vCd)9ea)af*Ol1*JCfnp$%Uu}%OuoN7g2}3C@`L5FlP#(sA=|h@iixuZC?qp^ z=L$=v$ZoI}|87Wh=&h7udff{aieKr*l+zDp?pf)_bbRvUf>kn;HCDMXNlgbbo!QRK I1x7am0No)LiU0rr delta 584 zcmexzm1*ZyrVZJAexH5Moc8h7)w{^+t*dqJ%=yhh23L$9JpFV=_k`zJ-?Q4DI*eSe z+ES)HSrVnWLtJ&)lO%hRkV9zl5qqWRt0e;bb zPPo`)y?HTAyZI&u&X<|2$FDHCf4;!v8}p=?Tm`^F0`u(|1ttf~&t$qP3KUSD>@TJQ zRwJ}Pim6NzEc8KA6)e;S6gs8=7IIL8sQL*MYEuRYO;Uj<%3UbMbV&^&!Zvx+LKmjT z8Zch6rYP7Tw?$Hn(UTJwWiS=$f{lB(C=e*%usDV})0AQIK~sat=ND@+Gg*Pyij!rR z*fa02W|%BsV++>4W{DKDGSIUEHd2$P+8ct!RF+CHDowUuTEZOZ%rJSQv*qOXOSPDN zT|sP-$p*_3ncsWB*qoD7JQcyZ9xan%cJP6Tb4-?AZpr*F6v98hoNaPJm@HV`yya5N z))6pqFXn@}P(3T0nEzM8*c_9KtE9o|_pFd&K35GBXP^9Kg(b6GH-z8S4GDzIl~T+b zdLd#meKKHu$5u))8cu$=GKINkGDPOUD)!0$C(BH(U!}!-e;Q0ok8Sc?V1zRO04>ts AA^-pY diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 37f853b1..ca025c83 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index faf93008..23d15a93 100755 --- a/gradlew +++ b/gradlew @@ -114,7 +114,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -213,7 +213,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 9b42019c..5eed7ee8 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,11 @@ goto fail :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar +set CLASSPATH= @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell From f1a1a72a024cb45782780afe4c65716f57816791 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:05:49 +0000 Subject: [PATCH 03/79] chore(deps): update dependency gg.jte:jte to v3.2.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 314b84dc..7a9d9647 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ dependencies { implementation group: "com.networknt", name: "json-schema-validator", version: "1.5.6" implementation 'org.glassfish:jakarta.json:2.0.1' //JTE for template processing - implementation('gg.jte:jte:3.2.0') + implementation('gg.jte:jte:3.2.1') implementation("org.freemarker:freemarker:2.3.34") } From 02fe2c859edf835fa7e8cccd4ee57b2615841fbd Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 12:39:35 +0200 Subject: [PATCH 04/79] docs: convert minimal example from README to a test --- README.md | 65 +-------------- .../datamanager/ro_crate/HelpFunctions.java | 7 ++ .../ExamplesOfSpecificationV1p1Test.java | 82 +++++++++++++++++++ .../spec-v1.1-example-json-files/minimal.json | 27 ++++++ 4 files changed, 119 insertions(+), 62 deletions(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java create mode 100644 src/test/resources/spec-v1.1-example-json-files/minimal.json diff --git a/README.md b/README.md index e5613a23..057fa2f7 100644 --- a/README.md +++ b/README.md @@ -196,71 +196,12 @@ boolean valid = validator.validate(crate); ## Adapting the specification examples -This section describes how to generate the [official specifications examples](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#minimal-example-of-ro-crate). Each example first shows the ro-crate-metadata.json and, below that, the required Java code to generate it. +We have an [example module with unit tests](src/test/java/edu/kit/datamanager/ro_crate/example/), describing how to generate the [official specifications examples](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#minimal-example-of-ro-crate). +Specifically, the examples for the Specification in version 1.1 are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). ### [Minimal example](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#minimal-example-of-ro-crate) -```json -{ "@context": "https://w3id.org/ro/crate/1.1/context", - "@graph": [ - - { - "@type": "CreativeWork", - "@id": "ro-crate-metadata.json", - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, - "about": {"@id": "./"} - }, - { - "@id": "./", - "identifier": "https://doi.org/10.4225/59/59672c09f4a4b", - "@type": "Dataset", - "datePublished": "2017", - "name": "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...", - "description": "Palliative care planning for nursing home residents with advanced dementia ...", - "license": {"@id": "https://creativecommons.org/licenses/by-nc-sa/3.0/au/"} - }, - { - "@id": "https://creativecommons.org/licenses/by-nc-sa/3.0/au/", - "@type": "CreativeWork", - "description": "This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Australia License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.", - "identifier": "https://creativecommons.org/licenses/by-nc-sa/3.0/au/", - "name": "Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)" - } - ] -} -``` - -Here, everything is created manually. -For the following examples, more convenient creation methods are used. - -```java - RoCrate crate = new RoCrate(); - - ContextualEntity license = new ContextualEntity.ContextualEntityBuilder() - .addType("CreativeWork") - .setId("https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addProperty("description", "This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Australia License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.") - .addProperty("identifier", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addProperty("name", "Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)") - .build(); - - crate.setRootDataEntity(new RootDataEntity.RootDataEntityBuilder() - .addProperty("identifier", "https://doi.org/10.4225/59/59672c09f4a4b") - .addProperty("datePublished", "2017") - .addProperty("name", "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...") - .addProperty("description", "Palliative care planning for nursing home residents with advanced dementia ...") - .setLicense(license) - .build()); - - crate.setJsonDescriptor(new ContextualEntity.ContextualEntityBuilder() - .setId("ro-crate-metadata.json") - .addType("CreativeWork") - .addIdProperty("about", "./") - .addIdProperty("conformsTo", "https://w3id.org/ro/crate/1.1") - .build() - ); - crate.addContextualEntity(license); -``` +see unit test ### [Example with files](https://www.researchobject.org/ro-crate/1.1/data-entities.html#example-linking-to-a-file-and-folders) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java index 8f32213f..97d29c58 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java @@ -80,6 +80,13 @@ public static void compareCrateJsonToFileInResources(File file1, File file2) thr compare(node1, node2, true); } + /** + * Compares the JSON metadata of a Crate object with a JSON file in the resources directory. + * + * @param crate1 The Crate object to compare. + * @param jsonFileString The path to the JSON file in the resources directory. + * @throws IOException If an error occurs while reading the JSON file. + */ public static void compareCrateJsonToFileInResources(Crate crate1, String jsonFileString) throws IOException { InputStream inputStream = HelpFunctions.class.getResourceAsStream( jsonFileString); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java new file mode 100644 index 00000000..c3997823 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -0,0 +1,82 @@ +package edu.kit.datamanager.ro_crate.examples; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; +import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; +import org.junit.jupiter.api.Test; +import org.opentest4j.AssertionFailedError; + +import java.io.IOException; + +/** + * This class contains examples of the RO-Crate specification version 1.1. + */ +public class ExamplesOfSpecificationV1p1Test { + + /** + * From: + * Minimal Example + * + *

+ * In this example, the crate is created without the builder. + * For the following examples, more convenient methods are used. + */ + @Test + void testMinimalCrateWithoutCrateBuilder() { + RoCrate minimal = new RoCrate(); + + ContextualEntity license = new ContextualEntity.ContextualEntityBuilder() + .addType("CreativeWork") + .setId("https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addProperty("description", "This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Australia License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.") + .addProperty("identifier", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addProperty("name", "Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)") + .build(); + + minimal.setRootDataEntity(new RootDataEntity.RootDataEntityBuilder() + .addProperty("identifier", "https://doi.org/10.4225/59/59672c09f4a4b") + .addProperty("datePublished", "2017") + .addProperty("name", "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...") + .addProperty("description", "Palliative care planning for nursing home residents with advanced dementia ...") + .setLicense(license) + .build()); + + minimal.setJsonDescriptor(new ContextualEntity.ContextualEntityBuilder() + .setId("ro-crate-metadata.json") + .addType("CreativeWork") + .addIdProperty("about", "./") + .addIdProperty("conformsTo", "https://w3id.org/ro/crate/1.1") + .build() + ); + minimal.addContextualEntity(license); + + // Print resulting json to console + prettyPrintJsonString(minimal.getJsonMetadata()); + // Compare with the example from the specification + try { + HelpFunctions.compareCrateJsonToFileInResources(minimal, "/spec-v1.1-example-json-files/minimal.json"); + } catch (IOException e) { + throw new AssertionFailedError("Missing resources file!", e); + } + } + + protected static void prettyPrintJsonString(String minimalJsonMetadata) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(minimalJsonMetadata); + // Enable pretty printing + String prettyJson = objectMapper + .enable(SerializationFeature.INDENT_OUTPUT) + .writeValueAsString(jsonNode); + // Print the pretty JSON + System.out.println(prettyJson); + } catch (JsonProcessingException e) { + throw new AssertionFailedError("Not able to process string as JSON!", e); + } + } +} diff --git a/src/test/resources/spec-v1.1-example-json-files/minimal.json b/src/test/resources/spec-v1.1-example-json-files/minimal.json new file mode 100644 index 00000000..3aee3b8c --- /dev/null +++ b/src/test/resources/spec-v1.1-example-json-files/minimal.json @@ -0,0 +1,27 @@ +{ "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + + { + "@type": "CreativeWork", + "@id": "ro-crate-metadata.json", + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, + "about": {"@id": "./"} + }, + { + "@id": "./", + "identifier": "https://doi.org/10.4225/59/59672c09f4a4b", + "@type": "Dataset", + "datePublished": "2017", + "name": "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...", + "description": "Palliative care planning for nursing home residents with advanced dementia ...", + "license": {"@id": "https://creativecommons.org/licenses/by-nc-sa/3.0/au/"} + }, + { + "@id": "https://creativecommons.org/licenses/by-nc-sa/3.0/au/", + "@type": "CreativeWork", + "description": "This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Australia License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.", + "identifier": "https://creativecommons.org/licenses/by-nc-sa/3.0/au/", + "name": "Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)" + } + ] +} \ No newline at end of file From d43edf78ec23a9cbad63edbd9aa2a4941c841b55 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 13:20:14 +0200 Subject: [PATCH 05/79] fix: avoid adding the same id property multiple times --- .../edu/kit/datamanager/ro_crate/RoCrate.java | 5 ++ .../ro_crate/entities/AbstractEntity.java | 85 ++++++++++++------- .../ExamplesOfSpecificationV1p1Test.java | 39 ++++++++- 3 files changed, 99 insertions(+), 30 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java index 330135b7..80cd4ff2 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java @@ -354,6 +354,11 @@ public RoCrateBuilder addName(String name) { return this; } + public RoCrateBuilder addIdentifier(String identifier) { + this.rootDataEntity.addProperty("identifier", identifier.strip()); + return this; + } + public RoCrateBuilder addDescription(String description) { this.rootDataEntity.addProperty(PROPERTY_DESCRIPTION, description); return this; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 580e625b..f963f42a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -239,33 +239,60 @@ private static boolean addProperty(ObjectNode whereToAdd, String key, JsonNode v * @param id the "id" of the property. */ public void addIdProperty(String name, String id) { - JsonNode jsonNode = addToIdProperty(name, id, this.properties.get(name)); - if (jsonNode != null) { - this.linkedTo.add(id); - this.properties.set(name, jsonNode); - this.notifyObservers(); - } + mergeIdIntoValue(id, this.properties.get(name)) + .ifPresent(newValue -> { + this.linkedTo.add(id); + this.properties.set(name, newValue); + this.notifyObservers(); + }); } - private static JsonNode addToIdProperty(String name, String id, JsonNode property) { - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - if (name != null && id != null) { - if (property == null) { - return objectMapper.createObjectNode().put("@id", id); - } else { - if (property.isArray()) { - ArrayNode ns = (ArrayNode) property; - ns.add(objectMapper.createObjectNode().put("@id", id)); - return ns; - } else { - ArrayNode newNodes = objectMapper.createArrayNode(); - newNodes.add(property); - newNodes.add(objectMapper.createObjectNode().put("@id", id)); - return newNodes; - } - } + /** + * Merges the given id into the current value, + * using this representation: {"@id" : "id"}. + *

+ * The current value can be null without errors. + * Only the id will be considered in this case. + *

+ * If the id is null-ish, it will not be added, similar to a null-ish value. + * If the id is already present, nothing will be done. + * If it is not an array and the id is not present, an array will be applied. + * + * @param id the id to add. + * @param currentValue the current value of the property. + * @return The updated value of the property. + * Empty if value does not change! + */ + private static Optional mergeIdIntoValue(String id, JsonNode currentValue) { + if (id == null || id.isBlank()) { return Optional.empty(); } + + ObjectMapper jsonBuilder = MyObjectMapper.getMapper(); + ObjectNode newIdObject = jsonBuilder.createObjectNode().put("@id", id); + if (currentValue == null || currentValue.isNull() || currentValue.isMissingNode()) { + return Optional.ofNullable(newIdObject); + } + + boolean isIdAlready = currentValue.asText().equals(id); + boolean isIdObjectAlready = currentValue.path("@id").asText().equals(id); + boolean isArrayWithIdPresent = currentValue.valueStream() + .anyMatch(node -> node + .path("@id") + .asText() + .equals(id)); + if (isIdAlready || isIdObjectAlready || isArrayWithIdPresent) { + return Optional.empty(); + } + + if (currentValue.isArray() && currentValue instanceof ArrayNode currentValueAsArray) { + currentValueAsArray.add(newIdObject); + return Optional.of(currentValueAsArray); + } else { + // property is not an array, so we make it an array + ArrayNode newNodes = jsonBuilder.createArrayNode(); + newNodes.add(currentValue); + newNodes.add(newIdObject); + return Optional.of(newNodes); } - return null; } /** @@ -486,11 +513,11 @@ public T addProperty(String key, boolean value) { * @return the generic builder */ public T addIdProperty(String name, String id) { - JsonNode jsonNode = AbstractEntity.addToIdProperty(name, id, this.properties.get(name)); - if (jsonNode != null) { - this.properties.set(name, jsonNode); - this.relatedItems.add(id); - } + AbstractEntity.mergeIdIntoValue(id, this.properties.get(name)) + .ifPresent(newValue -> { + this.properties.set(name, newValue); + this.relatedItems.add(id); + }); return self(); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index c3997823..8d1525c9 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -18,13 +18,50 @@ */ public class ExamplesOfSpecificationV1p1Test { + /** + * From: + * Minimal Example + * + *

+ * This is equivalent to {@link #testMinimalCrateWithoutCrateBuilder()}, but using more convenient APIs. + */ + @Test + void testMinimalCrateConvenient() { + // Example 1: Basic RO-Crate + RoCrate minimal = new RoCrate.RoCrateBuilder( + "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...", + "Palliative care planning for nursing home residents with advanced dementia ...", + "2017", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ) + .setLicense( new ContextualEntity.ContextualEntityBuilder() + .addType("CreativeWork") + .setId("https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addProperty("description", "This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Australia License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.") + .addProperty("identifier", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addProperty("name", "Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)") + .build() + ) + .addIdentifier("https://doi.org/10.4225/59/59672c09f4a4b") + .build(); + + // So you get something to see + prettyPrintJsonString(minimal.getJsonMetadata()); + // Compare with the example from the specification + try { + HelpFunctions.compareCrateJsonToFileInResources(minimal, "/spec-v1.1-example-json-files/minimal.json"); + } catch (IOException e) { + throw new AssertionFailedError("Missing resources file!", e); + } + } + /** * From: * Minimal Example * *

* In this example, the crate is created without the builder. - * For the following examples, more convenient methods are used. + * Otherwise, the example is the same as {@link #testMinimalCrateConvenient()}. */ @Test void testMinimalCrateWithoutCrateBuilder() { From ac069859eb0a2e459ee1c31ad643d3ea6646f13d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 13:20:51 +0200 Subject: [PATCH 06/79] cleanup: clean up unused imports and improve code formatting --- src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java | 8 +------- .../java/edu/kit/datamanager/ro_crate/HelpFunctions.java | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java index 80cd4ff2..e794974c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java @@ -10,13 +10,10 @@ import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; -import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity; import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity.DataEntityBuilder; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; + import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; import edu.kit.datamanager.ro_crate.externalproviders.dataentities.ImportFromDataCite; -import edu.kit.datamanager.ro_crate.externalproviders.organizationprovider.RorProvider; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import edu.kit.datamanager.ro_crate.payload.CratePayload; import edu.kit.datamanager.ro_crate.payload.RoCratePayload; @@ -26,12 +23,9 @@ import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions; import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; import java.io.File; import java.net.URI; -import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; import java.util.stream.StreamSupport; diff --git a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java index 97d29c58..d1d4aab6 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java @@ -34,8 +34,7 @@ public static void compareEntityWithFile(AbstractEntity entity, String string) t public static void compare(JsonNode node1, JsonNode node2, Boolean equals) { var comparator = new JsonComparator() { - public boolean compareValues(Object expected, Object actual) { - + public boolean compareValues(Object expected, Object actual) { return expected.equals(actual); } @@ -43,6 +42,7 @@ public boolean compareFields(String expected, String actual) { return expected.equals(actual); } }; + if (equals) { JSONCompare.assertMatches(node1, node2, comparator); } else { From e9a3bd501f337bfedf06a03193104aa9b9407c7f Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 13:40:41 +0200 Subject: [PATCH 07/79] ci: always run CI on pull requests --- .github/workflows/gradle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ba8a9308..fe8a5ce3 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -11,7 +11,6 @@ on: push: branches: [ main, development ] pull_request: - branches: [ main, development ] workflow_dispatch: env: From d47f90dd36f9d3ff9059bddc3d04bf34adfd8f81 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 13:57:44 +0200 Subject: [PATCH 08/79] refactor(tests): factor out helper methods --- .../datamanager/ro_crate/HelpFunctions.java | 28 +++++++++++++ .../ExamplesOfSpecificationV1p1Test.java | 42 ++----------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java index d1d4aab6..2c733d3e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions; @@ -11,6 +12,7 @@ import org.apache.commons.io.FileUtils; import io.json.compare.JSONCompare; import io.json.compare.JsonComparator; +import org.opentest4j.AssertionFailedError; import java.io.File; import java.io.IOException; @@ -73,6 +75,32 @@ public static void compareTwoCrateJson(Crate crate1, Crate crate2) throws JsonPr compare(node1, node2, true); } + public static void prettyPrintJsonString(String minimalJsonMetadata) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(minimalJsonMetadata); + // Enable pretty printing + String prettyJson = objectMapper + .enable(SerializationFeature.INDENT_OUTPUT) + .writeValueAsString(jsonNode); + // Print the pretty JSON + System.out.println(prettyJson); + } catch (JsonProcessingException e) { + throw new AssertionFailedError("Not able to process string as JSON!", e); + } + } + + public static void printAndAssertEquals(RoCrate crate, String pathToResource) { + // So you get something to see + prettyPrintJsonString(crate.getJsonMetadata()); + // Compare with the example from the specification + try { + HelpFunctions.compareCrateJsonToFileInResources(crate, pathToResource); + } catch (IOException e) { + throw new AssertionFailedError("Missing resources file!", e); + } + } + public static void compareCrateJsonToFileInResources(File file1, File file2) throws IOException { ObjectMapper objectMapper = MyObjectMapper.getMapper(); JsonNode node1 = JsonUtilFunctions.unwrapSingleArray(objectMapper.readTree(file1)); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index 8d1525c9..0e785bc2 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -1,17 +1,12 @@ package edu.kit.datamanager.ro_crate.examples; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; + import org.junit.jupiter.api.Test; -import org.opentest4j.AssertionFailedError; -import java.io.IOException; +import static edu.kit.datamanager.ro_crate.HelpFunctions.printAndAssertEquals; /** * This class contains examples of the RO-Crate specification version 1.1. @@ -45,14 +40,7 @@ void testMinimalCrateConvenient() { .addIdentifier("https://doi.org/10.4225/59/59672c09f4a4b") .build(); - // So you get something to see - prettyPrintJsonString(minimal.getJsonMetadata()); - // Compare with the example from the specification - try { - HelpFunctions.compareCrateJsonToFileInResources(minimal, "/spec-v1.1-example-json-files/minimal.json"); - } catch (IOException e) { - throw new AssertionFailedError("Missing resources file!", e); - } + printAndAssertEquals(minimal, "/spec-v1.1-example-json-files/minimal.json"); } /** @@ -92,28 +80,6 @@ void testMinimalCrateWithoutCrateBuilder() { ); minimal.addContextualEntity(license); - // Print resulting json to console - prettyPrintJsonString(minimal.getJsonMetadata()); - // Compare with the example from the specification - try { - HelpFunctions.compareCrateJsonToFileInResources(minimal, "/spec-v1.1-example-json-files/minimal.json"); - } catch (IOException e) { - throw new AssertionFailedError("Missing resources file!", e); - } - } - - protected static void prettyPrintJsonString(String minimalJsonMetadata) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(minimalJsonMetadata); - // Enable pretty printing - String prettyJson = objectMapper - .enable(SerializationFeature.INDENT_OUTPUT) - .writeValueAsString(jsonNode); - // Print the pretty JSON - System.out.println(prettyJson); - } catch (JsonProcessingException e) { - throw new AssertionFailedError("Not able to process string as JSON!", e); - } + printAndAssertEquals(minimal, "/spec-v1.1-example-json-files/minimal.json"); } } From 0c4321188fd9651f31d08ea766c988020692349a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 14:04:51 +0200 Subject: [PATCH 09/79] docs: update examples in ExamplesOfSpecificationV1p1Test with repository links --- .../ro_crate/examples/ExamplesOfSpecificationV1p1Test.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index 0e785bc2..2ad9334a 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -10,19 +10,20 @@ /** * This class contains examples of the RO-Crate specification version 1.1. + *

+ * This is supposed to serve both as a user guide and as a test for the implementation. */ public class ExamplesOfSpecificationV1p1Test { /** * From: * Minimal Example - * + * (location in repo) *

* This is equivalent to {@link #testMinimalCrateWithoutCrateBuilder()}, but using more convenient APIs. */ @Test void testMinimalCrateConvenient() { - // Example 1: Basic RO-Crate RoCrate minimal = new RoCrate.RoCrateBuilder( "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...", "Palliative care planning for nursing home residents with advanced dementia ...", @@ -46,7 +47,7 @@ void testMinimalCrateConvenient() { /** * From: * Minimal Example - * + * (location in repo) *

* In this example, the crate is created without the builder. * Otherwise, the example is the same as {@link #testMinimalCrateConvenient()}. From 424b8f6a791c394193677597ae6c06ac8344ab6b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 14:47:44 +0200 Subject: [PATCH 10/79] chore: improve readability of test to make it more of a guide --- .../examples/ExamplesOfSpecificationV1p1Test.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index 2ad9334a..e4d14700 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -12,6 +12,7 @@ * This class contains examples of the RO-Crate specification version 1.1. *

* This is supposed to serve both as a user guide and as a test for the implementation. + * Executing a test may also print some interesting information to the console. */ public class ExamplesOfSpecificationV1p1Test { @@ -24,17 +25,20 @@ public class ExamplesOfSpecificationV1p1Test { */ @Test void testMinimalCrateConvenient() { + String licenseID = "https://creativecommons.org/licenses/by-nc-sa/3.0/au/"; RoCrate minimal = new RoCrate.RoCrateBuilder( "Data files associated with the manuscript:Effects of facilitated family case conferencing for ...", "Palliative care planning for nursing home residents with advanced dementia ...", "2017", - "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + licenseID ) + // We already had to set the license ID in the builder, + // but we can override it with more details to fit the example: .setLicense( new ContextualEntity.ContextualEntityBuilder() .addType("CreativeWork") - .setId("https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .setId(licenseID) .addProperty("description", "This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Australia License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.") - .addProperty("identifier", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addProperty("identifier", licenseID) .addProperty("name", "Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)") .build() ) @@ -72,6 +76,8 @@ void testMinimalCrateWithoutCrateBuilder() { .setLicense(license) .build()); + // This is pretty low-level. We are considering hiding/replacing this detailed API in major versions, + // so tell us (for example, open an issue) if you have a use case for it! minimal.setJsonDescriptor(new ContextualEntity.ContextualEntityBuilder() .setId("ro-crate-metadata.json") .addType("CreativeWork") From 375e53a0b0d85f9998360f7f3ed6311a49cf81db Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 14:48:36 +0200 Subject: [PATCH 11/79] docs: add javadocs to RoCrateBuilder.addIdentifier --- src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java index e794974c..356b0159 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java @@ -348,6 +348,13 @@ public RoCrateBuilder addName(String name) { return this; } + /** + * Adds an "identifier" property to the root data entity. + *

+ * This is useful e.g. to assign e.g. a DOI to this crate. + * @param identifier the identifier to add. + * @return this builder. + */ public RoCrateBuilder addIdentifier(String identifier) { this.rootDataEntity.addProperty("identifier", identifier.strip()); return this; From 27d671e21dccd8d73d89018002df49805581481d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 15:13:06 +0200 Subject: [PATCH 12/79] docs: convert files-and-folders example from README to a test --- README.md | 67 +------------------ .../ExamplesOfSpecificationV1p1Test.java | 45 +++++++++++++ .../files-and-folders.json | 38 +++++++++++ 3 files changed, 84 insertions(+), 66 deletions(-) create mode 100644 src/test/resources/spec-v1.1-example-json-files/files-and-folders.json diff --git a/README.md b/README.md index 057fa2f7..5b490173 100644 --- a/README.md +++ b/README.md @@ -205,72 +205,7 @@ see unit test ### [Example with files](https://www.researchobject.org/ro-crate/1.1/data-entities.html#example-linking-to-a-file-and-folders) -```json -{ "@context": "https://w3id.org/ro/crate/1.1/context", - "@graph": [ - { - "@type": "CreativeWork", - "@id": "ro-crate-metadata.json", - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, - "about": {"@id": "./"} - }, - { - "@id": "./", - "@type": [ - "Dataset" - ], - "hasPart": [ - { - "@id": "cp7glop.ai" - }, - { - "@id": "lots_of_little_files/" - } - ] - }, - { - "@id": "cp7glop.ai", - "@type": "File", - "name": "Diagram showing trend to increase", - "contentSize": "383766", - "description": "Illustrator file for Glop Pot", - "encodingFormat": "application/pdf" - }, - { - "@id": "lots_of_little_files/", - "@type": "Dataset", - "name": "Too many files", - "description": "This directory contains many small files, that we're not going to describe in detail." - } - ] -} -``` - -Here we use the inner builder classes for the construction of the crate. -Doing so, the Metadata File Descriptor and the Root Data Entity entities are added automatically. -`setSource()` is used to provide the actual location of these Data Entities (if they are not remote). -The Data Entity file in the crate will have the name of the entity's ID. - -```java - RoCrate crate = new RoCrate.RoCrateBuilder() - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addContent (Paths.get("path to file"), "cp7glop.ai") - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addContent (Paths.get("path_to_files"), "lots_of_little_files/") - .addProperty("name", "Too many files") - .addProperty("description", "This directory contains many small files, that we're not going to describe in detail.") - .build() - ) - .build(); -``` +see unit test ### [Example with web resources](https://www.researchobject.org/ro-crate/1.1/data-entities.html#web-based-data-entities) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index e4d14700..a9b24d7c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -2,10 +2,14 @@ import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; +import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; import org.junit.jupiter.api.Test; +import java.nio.file.Paths; + import static edu.kit.datamanager.ro_crate.HelpFunctions.printAndAssertEquals; /** @@ -89,4 +93,45 @@ void testMinimalCrateWithoutCrateBuilder() { printAndAssertEquals(minimal, "/spec-v1.1-example-json-files/minimal.json"); } + + // https://www.researchobject.org/ro-crate/specification/1.1/data-entities.html#example-linking-to-a-file-and-folders + + /** + * From: + * "Example linking to a file and folders" + * (location in repo) + *

+ * Here we use the inner builder classes for the construction of the crate. + * Doing so, the Metadata File Descriptor and the Root Data Entity entities are added automatically. + * `setSource()` is used to provide the actual location of these Data Entities (if they are not remote). + * The Data Entity file in the crate will have the name of the entity's ID. + */ + @Test + void testLinkingToFileAndFolders() { + RoCrate crate = new RoCrate.RoCrateBuilder() + .addDataEntity( + new FileEntity.FileEntityBuilder() + // This will tell us where the file is located. It will be copied to the crate. + .setLocation(Paths.get("path to file")) + // If no ID is given explicitly, the ID will be set to the filename. + // Changing the ID means also to set the file name within the crate! + .setId("cp7glop.ai") + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .setLocation(Paths.get("path_to_files")) + .setId("lots_of_little_files/") + .addProperty("name", "Too many files") + .addProperty("description", "This directory contains many small files, that we're not going to describe in detail.") + .build() + ) + .build(); + + printAndAssertEquals(crate, "/spec-v1.1-example-json-files/files-and-folders.json"); + } } diff --git a/src/test/resources/spec-v1.1-example-json-files/files-and-folders.json b/src/test/resources/spec-v1.1-example-json-files/files-and-folders.json new file mode 100644 index 00000000..e8bb2ab0 --- /dev/null +++ b/src/test/resources/spec-v1.1-example-json-files/files-and-folders.json @@ -0,0 +1,38 @@ +{ "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@type": "CreativeWork", + "@id": "ro-crate-metadata.json", + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, + "about": {"@id": "./"} + }, + { + "@id": "./", + "@type": [ + "Dataset" + ], + "hasPart": [ + { + "@id": "cp7glop.ai" + }, + { + "@id": "lots_of_little_files/" + } + ] + }, + { + "@id": "cp7glop.ai", + "@type": "File", + "name": "Diagram showing trend to increase", + "contentSize": "383766", + "description": "Illustrator file for Glop Pot", + "encodingFormat": "application/pdf" + }, + { + "@id": "lots_of_little_files/", + "@type": "Dataset", + "name": "Too many files", + "description": "This directory contains many small files, that we're not going to describe in detail." + } + ] +} From f24a7a63e74a7304cb49774c0896d48b3650126b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 15:13:28 +0200 Subject: [PATCH 13/79] docs: enhance Javadoc for id property in AbstractEntity to clarify ID usage --- .../edu/kit/datamanager/ro_crate/entities/AbstractEntity.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index f963f42a..8f01f62c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -397,6 +397,10 @@ protected String getId() { * Setting the id property of the entity, if the given value is not * null. If the id is not encoded, the encoding will be done. * + * @apiNote IDs are not just names! The ID may have effects + * on parts of your crate! For example: If the entity represents a + * file which will be copied into the crate, writers must use the + * ID as filename. * @param id the String representing the id. * @return the generic builder. */ From 917b89f54479b71f7093d121962fbd43a1bb59a9 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 16:03:55 +0200 Subject: [PATCH 14/79] fix: javadoc compilation with java 17 does not support @apiNote --- .../kit/datamanager/ro_crate/entities/AbstractEntity.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 8f01f62c..8699fd5b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -396,11 +396,12 @@ protected String getId() { /** * Setting the id property of the entity, if the given value is not * null. If the id is not encoded, the encoding will be done. - * - * @apiNote IDs are not just names! The ID may have effects + *

+ * NOTE: IDs are not just names! The ID may have effects * on parts of your crate! For example: If the entity represents a * file which will be copied into the crate, writers must use the * ID as filename. + * * @param id the String representing the id. * @return the generic builder. */ From b551a8c6c502d5f5a7dedcd6f240f55464a72c44 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 16:17:26 +0200 Subject: [PATCH 15/79] docs: enhance Javadoc in ExamplesOfSpecificationV1p1Test to introduce the reader better --- .../ExamplesOfSpecificationV1p1Test.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index a9b24d7c..28391569 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -6,6 +6,7 @@ import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; import org.junit.jupiter.api.Test; import java.nio.file.Paths; @@ -25,7 +26,11 @@ public class ExamplesOfSpecificationV1p1Test { * Minimal Example * (location in repo) *

- * This is equivalent to {@link #testMinimalCrateWithoutCrateBuilder()}, but using more convenient APIs. + * This example produces a minimal crate with a + * name, description, date, license and identifier. + *

+ * This example produces the same result as + * {@link #testMinimalCrateWithoutCrateBuilder()}, but using more convenient APIs. */ @Test void testMinimalCrateConvenient() { @@ -57,8 +62,9 @@ void testMinimalCrateConvenient() { * Minimal Example * (location in repo) *

- * In this example, the crate is created without the builder. - * Otherwise, the example is the same as {@link #testMinimalCrateConvenient()}. + * In this example, the minimal crate is created without the builder. + * This should only be done if necessary: Use the builder if possible. + * This example produces the same result as {@link #testMinimalCrateConvenient()}. */ @Test void testMinimalCrateWithoutCrateBuilder() { @@ -101,10 +107,16 @@ void testMinimalCrateWithoutCrateBuilder() { * "Example linking to a file and folders" * (location in repo) *

- * Here we use the inner builder classes for the construction of the crate. - * Doing so, the Metadata File Descriptor and the Root Data Entity entities are added automatically. - * `setSource()` is used to provide the actual location of these Data Entities (if they are not remote). - * The Data Entity file in the crate will have the name of the entity's ID. + * This example adds a File(Entity) and a DataSet(Entity) to the crate. + * The file and the folder are referenced by their location. This way + * they will be copied to the crate when writing it using a + * {@link CrateWriter}. + * The name of the file and the folder will be implicitly set to the + * ID of the respective entity in order to conform to the specification. + *

+ * Here we use the inner builder classes for the construction of the + * crate. In contrast to {@link #testMinimalCrateWithoutCrateBuilder()}, + * we do not have to care about specification details. */ @Test void testLinkingToFileAndFolders() { From d35c1768470cdb6d13311cd98bdd32c8af6338c7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 16:34:48 +0200 Subject: [PATCH 16/79] docs: convert web-based-data-entities example from README to a test --- README.md | 66 +------------------ .../ExamplesOfSpecificationV1p1Test.java | 36 ++++++++++ .../web-based-data-entities.json | 39 +++++++++++ 3 files changed, 76 insertions(+), 65 deletions(-) create mode 100644 src/test/resources/spec-v1.1-example-json-files/web-based-data-entities.json diff --git a/README.md b/README.md index 5b490173..1ca4469d 100644 --- a/README.md +++ b/README.md @@ -209,71 +209,7 @@ see unit test ### [Example with web resources](https://www.researchobject.org/ro-crate/1.1/data-entities.html#web-based-data-entities) -```json -{ "@context": "https://w3id.org/ro/crate/1.1/context", - "@graph": [ - { - "@type": "CreativeWork", - "@id": "ro-crate-metadata.json", - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, - "about": {"@id": "./"} - }, - { - "@id": "./", - "@type": [ - "Dataset" - ], - "hasPart": [ - { - "@id": "survey-responses-2019.csv" - }, - { - "@id": "https://zenodo.org/record/3541888/files/ro-crate-1.0.0.pdf" - }, - ] - }, - { - "@id": "survey-responses-2019.csv", - "@type": "File", - "name": "Survey responses", - "contentSize": "26452", - "encodingFormat": "text/csv" - }, - { - "@id": "https://zenodo.org/record/3541888/files/ro-crate-1.0.0.pdf", - "@type": "File", - "name": "RO-Crate specification", - "contentSize": "310691", - "description": "RO-Crate specification", - "encodingFormat": "application/pdf" - } -] -} -``` - -The web resource does not use `.setSource()`, but uses the ID to indicate the file's location. - -```java - RoCrate crate = new RoCrate.RoCrateBuilder() - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addContent (Paths.get("README.md"), "survey-responses-2019.csv") - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .setEncodingFormat("text/csv") - .build() - ) - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addContent(URI.create("https://zenodo.org/record/3541888/files/ro-crate-1.0.0.pdf")) - .addProperty("name", "RO-Crate specification") - .addProperty("contentSize", "310691") - .addProperty("description", "RO-Crate specification") - .setEncodingFormat("application/pdf") - .build() - ) - .build(); -``` +see unit test ### [Example with file, author, location](https://www.researchobject.org/ro-crate/1.1/appendix/jsonld.html) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index 28391569..a48a225a 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -9,6 +9,7 @@ import edu.kit.datamanager.ro_crate.writer.CrateWriter; import org.junit.jupiter.api.Test; +import java.net.URI; import java.nio.file.Paths; import static edu.kit.datamanager.ro_crate.HelpFunctions.printAndAssertEquals; @@ -146,4 +147,39 @@ void testLinkingToFileAndFolders() { printAndAssertEquals(crate, "/spec-v1.1-example-json-files/files-and-folders.json"); } + + /** + * From: + * "Example linking to a file and folders" + * (location in repo) + *

+ * This example adds twp FileEntities to the crate. + * One is a local file, the other one is located in the web + * and will not be copied to the crate. + */ + @Test + void testWebBasedDataEntities() { + RoCrate crate = new RoCrate.RoCrateBuilder() + .addDataEntity( + new FileEntity.FileEntityBuilder() + .setLocation(Paths.get("README.md")) + .setId("survey-responses-2019.csv") + .addProperty("name", "Survey responses") + .addProperty("contentSize", "26452") + .setEncodingFormat("text/csv") + .build() + ) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .setLocation(URI.create("https://zenodo.org/record/3541888/files/ro-crate-1.0.0.pdf")) + .addProperty("name", "RO-Crate specification") + .addProperty("contentSize", "310691") + .addProperty("description", "RO-Crate specification") + .setEncodingFormat("application/pdf") + .build() + ) + .build(); + + printAndAssertEquals(crate, "/spec-v1.1-example-json-files/web-based-data-entities.json"); + } } diff --git a/src/test/resources/spec-v1.1-example-json-files/web-based-data-entities.json b/src/test/resources/spec-v1.1-example-json-files/web-based-data-entities.json new file mode 100644 index 00000000..d5245907 --- /dev/null +++ b/src/test/resources/spec-v1.1-example-json-files/web-based-data-entities.json @@ -0,0 +1,39 @@ +{ "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@type": "CreativeWork", + "@id": "ro-crate-metadata.json", + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, + "about": {"@id": "./"} + }, + { + "@id": "./", + "@type": [ + "Dataset" + ], + "hasPart": [ + { + "@id": "survey-responses-2019.csv" + }, + { + "@id": "https://zenodo.org/record/3541888/files/ro-crate-1.0.0.pdf" + } + ] + }, + { + "@id": "survey-responses-2019.csv", + "@type": "File", + "name": "Survey responses", + "contentSize": "26452", + "encodingFormat": "text/csv" + }, + { + "@id": "https://zenodo.org/record/3541888/files/ro-crate-1.0.0.pdf", + "@type": "File", + "name": "RO-Crate specification", + "contentSize": "310691", + "description": "RO-Crate specification", + "encodingFormat": "application/pdf" + } + ] +} From fe02a9fe1d7dd4e8508b76a5f364fdb001662eae Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 16:43:41 +0200 Subject: [PATCH 17/79] docs: fix example description for web-based data entities in ExamplesOfSpecificationV1p1Test --- .../ro_crate/examples/ExamplesOfSpecificationV1p1Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index a48a225a..285f9010 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -150,7 +150,7 @@ void testLinkingToFileAndFolders() { /** * From: - * "Example linking to a file and folders" + * Example with web-based data entities * (location in repo) *

* This example adds twp FileEntities to the crate. From 8ab6f1b1bc73af898020e8848838bf3838b48788 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 2 May 2025 16:55:30 +0200 Subject: [PATCH 18/79] docs: convert file-author-location example from README to a test --- README.md | 76 ------------------- .../ExamplesOfSpecificationV1p1Test.java | 47 ++++++++++++ .../file-author-location.json | 47 ++++++++++++ 3 files changed, 94 insertions(+), 76 deletions(-) create mode 100644 src/test/resources/spec-v1.1-example-json-files/file-author-location.json diff --git a/README.md b/README.md index 1ca4469d..b81fddda 100644 --- a/README.md +++ b/README.md @@ -213,86 +213,10 @@ see unit test ### [Example with file, author, location](https://www.researchobject.org/ro-crate/1.1/appendix/jsonld.html) -```json -{ "@context": "https://w3id.org/ro/crate/1.1/context", - "@graph": [ - - { - "@type": "CreativeWork", - "@id": "ro-crate-metadata.json", - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, - "about": {"@id": "./"}, - "description": "RO-Crate Metadata File Descriptor (this file)" - }, - { - "@id": "./", - "@type": "Dataset", - "name": "Example RO-Crate", - "description": "The RO-Crate Root Data Entity", - "datePublished": "2020", - "license": {"@id": "https://spdx.org/licenses/CC-BY-NC-SA-4.0"}, - "hasPart": [ - {"@id": "data1.txt"}, - {"@id": "data2.txt"} - ] - }, - { - "@id": "data1.txt", - "@type": "File", - "description": "One of hopefully many Data Entities", - "author": {"@id": "#alice"}, - "contentLocation": {"@id": "http://sws.geonames.org/8152662/"} - }, - { - "@id": "data2.txt", - "@type": "File" - }, - - { - "@id": "#alice", - "@type": "Person", - "name": "Alice", - "description": "One of hopefully many Contextual Entities" - }, - { - "@id": "http://sws.geonames.org/8152662/", - "@type": "Place", - "name": "Catalina Park" - } - ] -} -``` - If there is no special method for including relative entities (ID properties) one can use `.addIdProperty("key","value")`. ```java - PersonEntity alice = new PersonEntity.PersonEntityBuilder() - .setId("#alice") - .addProperty("name", "Alice") - .addProperty("description", "One of hopefully many Contextual Entities") - .build(); - PlaceEntity park = new PlaceEntity.PlaceEntityBuilder() - .addContent(URI.create("http://sws.geonames.org/8152662/")) - .addProperty("name", "Catalina Park") - .build(); - RoCrate crate = new RoCrate.RoCrateBuilder("Example RO-Crate", "The RO-Crate Root Data Entity", "2020", "https://spdx.org/licenses/CC-BY-NC-SA-4.0") - .addContextualEntity(park) - .addContextualEntity(alice) - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addContent(Paths.get("......."), "data2.txt") - .build() - ) - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addContent(Paths.get("......."), "data1.txt") - .addProperty("description", "One of hopefully many Data Entities") - .addAuthor(alice.getId()) - .addIdProperty("contentLocation", park) - .build() - ) - .build(); ``` ### [Example with computational workflow](https://www.researchobject.org/ro-crate/1.1/workflows.html#complete-workflow-example) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index 285f9010..bb45e35f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -2,6 +2,8 @@ import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; +import edu.kit.datamanager.ro_crate.entities.contextual.PersonEntity; +import edu.kit.datamanager.ro_crate.entities.contextual.PlaceEntity; import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; @@ -182,4 +184,49 @@ void testWebBasedDataEntities() { printAndAssertEquals(crate, "/spec-v1.1-example-json-files/web-based-data-entities.json"); } + + /** + * From: + * Example with file, author, and location + * (location in repo) + */ + @Test + void testWithFileAuthorLocation() { + PersonEntity alice = new PersonEntity.PersonEntityBuilder() + .setId("#alice") + .addProperty("name", "Alice") + .addProperty("description", "One of hopefully many Contextual Entities") + .build(); + PlaceEntity park = new PlaceEntity.PlaceEntityBuilder() + .setId(URI.create("http://sws.geonames.org/8152662/").toString()) + .addProperty("name", "Catalina Park") + .build(); + + RoCrate crate = new RoCrate.RoCrateBuilder( + "Example RO-Crate", + "The RO-Crate Root Data Entity", + "2020", + "https://spdx.org/licenses/CC-BY-NC-SA-4.0" + ) + .addContextualEntity(park) + .addContextualEntity(alice) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .setLocation(Paths.get(".......")) + .setId("data2.txt") + .build() + ) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .setLocation(Paths.get(".......")) + .setId("data1.txt") + .addProperty("description", "One of hopefully many Data Entities") + .addAuthor(alice.getId()) + .addIdProperty("contentLocation", park) + .build() + ) + .build(); + + printAndAssertEquals(crate, "/spec-v1.1-example-json-files/file-author-location.json"); + } } diff --git a/src/test/resources/spec-v1.1-example-json-files/file-author-location.json b/src/test/resources/spec-v1.1-example-json-files/file-author-location.json new file mode 100644 index 00000000..6bd83070 --- /dev/null +++ b/src/test/resources/spec-v1.1-example-json-files/file-author-location.json @@ -0,0 +1,47 @@ +{ "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + + { + "@type": "CreativeWork", + "@id": "ro-crate-metadata.json", + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, + "about": {"@id": "./"}, + "description": "RO-Crate Metadata File Descriptor (this file)" + }, + { + "@id": "./", + "@type": "Dataset", + "name": "Example RO-Crate", + "description": "The RO-Crate Root Data Entity", + "hasPart": [ + {"@id": "data1.txt"}, + {"@id": "data2.txt"} + ] + }, + + + { + "@id": "data1.txt", + "@type": "File", + "description": "One of hopefully many Data Entities", + "author": {"@id": "#alice"}, + "contentLocation": {"@id": "http://sws.geonames.org/8152662/"} + }, + { + "@id": "data2.txt", + "@type": "File" + }, + + { + "@id": "#alice", + "@type": "Person", + "name": "Alice", + "description": "One of hopefully many Contextual Entities" + }, + { + "@id": "http://sws.geonames.org/8152662/", + "@type": "Place", + "name": "Catalina Park" + } + ] +} From 8197ce808496f7ac3115c930b20bee1337de228e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Sat, 3 May 2025 00:33:37 +0200 Subject: [PATCH 19/79] fix: file-author-location test is missing license and datePublished. The example does not have it, so we need to remove it afterwards. --- README.md | 6 +-- .../ExamplesOfSpecificationV1p1Test.java | 41 +++++++++++++++++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b81fddda..19690008 100644 --- a/README.md +++ b/README.md @@ -213,12 +213,8 @@ see unit test ### [Example with file, author, location](https://www.researchobject.org/ro-crate/1.1/appendix/jsonld.html) -If there is no special method for including relative entities (ID properties) one can use `.addIdProperty("key","value")`. - -```java - +see unit test -``` ### [Example with computational workflow](https://www.researchobject.org/ro-crate/1.1/workflows.html#complete-workflow-example) ```json diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index bb45e35f..3c7ca0e7 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -1,9 +1,11 @@ package edu.kit.datamanager.ro_crate.examples; import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import edu.kit.datamanager.ro_crate.entities.contextual.PersonEntity; import edu.kit.datamanager.ro_crate.entities.contextual.PlaceEntity; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; @@ -189,24 +191,37 @@ void testWebBasedDataEntities() { * From: * Example with file, author, and location * (location in repo) + *

+ * This example shows how to connect entities. If there is no specific method like + * {@link DataEntity.DataEntityBuilder#addAuthor(String)} for referencing other + * entities, one can use the more generic + * {@link AbstractEntity.AbstractEntityBuilder#addIdProperty(String, AbstractEntity)} + * or {@link AbstractEntity.AbstractEntityBuilder#addIdProperty(String, String)}. + *

+ * Important Note! If you connect entities, make sure all entities are being + * added to the crate. We currently can't enforce this properly yet. */ @Test void testWithFileAuthorLocation() { - PersonEntity alice = new PersonEntity.PersonEntityBuilder() + // These two entities will be connected to others later on. Therefore, we make + // them easier referencable. Referencing can be done using the whole entity or + // its ID. + final PersonEntity alice = new PersonEntity.PersonEntityBuilder() .setId("#alice") .addProperty("name", "Alice") .addProperty("description", "One of hopefully many Contextual Entities") .build(); - PlaceEntity park = new PlaceEntity.PlaceEntityBuilder() + final PlaceEntity park = new PlaceEntity.PlaceEntityBuilder() .setId(URI.create("http://sws.geonames.org/8152662/").toString()) .addProperty("name", "Catalina Park") .build(); + final String licenseId = "https://spdx.org/licenses/CC-BY-NC-SA-4.0"; - RoCrate crate = new RoCrate.RoCrateBuilder( + final RoCrate crate = new RoCrate.RoCrateBuilder( "Example RO-Crate", "The RO-Crate Root Data Entity", "2020", - "https://spdx.org/licenses/CC-BY-NC-SA-4.0" + licenseId ) .addContextualEntity(park) .addContextualEntity(alice) @@ -221,12 +236,30 @@ void testWithFileAuthorLocation() { .setLocation(Paths.get(".......")) .setId("data1.txt") .addProperty("description", "One of hopefully many Data Entities") + // ↓ This is the specific way to add an author .addAuthor(alice.getId()) + // ↓ This is the generic way to add a location or other relations .addIdProperty("contentLocation", park) .build() ) .build(); + /* + The builder enforces to provide a license and a publishing date, + but the example does not have them. So we have to remove them below: + */ + + // **Note**: When you add a license, even if only by a string, the crate will + // implicitly also get a small ContextEntity for this license. When we remove + // this (any) entity, all references to it will be removed as well to ensure + // consistency within the crate. Therefore, there will be no trace left of + // the license. + crate.deleteEntityById(licenseId); + + // The datePublished property is a simple property and simple to remove without + // any further internal checks. + crate.getRootDataEntity().removeProperty("datePublished"); + printAndAssertEquals(crate, "/spec-v1.1-example-json-files/file-author-location.json"); } } From a8e8e2e0450414c156c4e587663992518cb331f4 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Sat, 3 May 2025 01:08:41 +0200 Subject: [PATCH 20/79] docs: add complete workflow example test --- README.md | 253 +----------------- .../ExamplesOfSpecificationV1p1Test.java | 150 ++++++++++- .../complete-workflow-example.json | 113 ++++++++ 3 files changed, 261 insertions(+), 255 deletions(-) create mode 100644 src/test/resources/spec-v1.1-example-json-files/complete-workflow-example.json diff --git a/README.md b/README.md index 19690008..edb7c12e 100644 --- a/README.md +++ b/README.md @@ -196,254 +196,5 @@ boolean valid = validator.validate(crate); ## Adapting the specification examples -We have an [example module with unit tests](src/test/java/edu/kit/datamanager/ro_crate/example/), describing how to generate the [official specifications examples](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#minimal-example-of-ro-crate). -Specifically, the examples for the Specification in version 1.1 are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). - -### [Minimal example](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#minimal-example-of-ro-crate) - -see unit test - -### [Example with files](https://www.researchobject.org/ro-crate/1.1/data-entities.html#example-linking-to-a-file-and-folders) - -see unit test - -### [Example with web resources](https://www.researchobject.org/ro-crate/1.1/data-entities.html#web-based-data-entities) - -see unit test - -### [Example with file, author, location](https://www.researchobject.org/ro-crate/1.1/appendix/jsonld.html) - -see unit test - -### [Example with computational workflow](https://www.researchobject.org/ro-crate/1.1/workflows.html#complete-workflow-example) - -```json -{ "@context": "https://w3id.org/ro/crate/1.1/context", - "@graph": [ - { - "@type": "CreativeWork", - "@id": "ro-crate-metadata.json", - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, - "about": {"@id": "./"} - }, - { - "@id": "./", - "@type": "Dataset", - "name": "Example RO-Crate", - "description": "The RO-Crate Root Data Entity", - "datePublished": "2020", - "license": {"@id": "https://spdx.org/licenses/CC-BY-NC-SA-4.0"}, - "hasPart": [ - { "@id": "workflow/alignment.knime" } - ] - }, - { - "@id": "workflow/alignment.knime", - "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], - "conformsTo": - {"@id": "https://bioschemas.org/profiles/ComputationalWorkflow/0.5-DRAFT-2020_07_21/"}, - "name": "Sequence alignment workflow", - "programmingLanguage": {"@id": "#knime"}, - "creator": {"@id": "#alice"}, - "dateCreated": "2020-05-23", - "license": { "@id": "https://spdx.org/licenses/CC-BY-NC-SA-4.0"}, - "input": [ - { "@id": "#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b"} - ], - "output": [ - { "@id": "#6c703fee-6af7-4fdb-a57d-9e8bc4486044"}, - { "@id": "#2f32b861-e43c-401f-8c42-04fd84273bdf"} - ], - "sdPublisher": {"@id": "#workflow-hub"}, - "url": "http://example.com/workflows/alignment", - "version": "0.5.0" - }, - { - "@id": "#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b", - "@type": "FormalParameter", - "conformsTo": {"@id": "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/"}, - "name": "genome_sequence", - "valueRequired": true, - "additionalType": {"@id": "http://edamontology.org/data_2977"}, - "format": {"@id": "http://edamontology.org/format_1929"} - }, - { - "@id": "#6c703fee-6af7-4fdb-a57d-9e8bc4486044", - "@type": "FormalParameter", - "conformsTo": {"@id": "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/"}, - "name": "cleaned_sequence", - "additionalType": {"@id": "http://edamontology.org/data_2977"}, - "encodingFormat": {"@id": "http://edamontology.org/format_2572"} - }, - { - "@id": "#2f32b861-e43c-401f-8c42-04fd84273bdf", - "@type": "FormalParameter", - "conformsTo": {"@id": "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/"}, - "name": "sequence_alignment", - "additionalType": {"@id": "http://edamontology.org/data_1383"}, - "encodingFormat": {"@id": "http://edamontology.org/format_1982"} - }, - { - "@id": "https://spdx.org/licenses/CC-BY-NC-SA-4.0", - "@type": "CreativeWork", - "name": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", - "alternateName": "CC-BY-NC-SA-4.0" - }, - { - "@id": "#knime", - "@type": "ProgrammingLanguage", - "name": "KNIME Analytics Platform", - "alternateName": "KNIME", - "url": "https://www.knime.com/whats-new-in-knime-41", - "version": "4.1.3" - }, - { - "@id": "#alice", - "@type": "Person", - "name": "Alice Brown" - }, - { - "@id": "#workflow-hub", - "@type": "Organization", - "name": "Example Workflow Hub", - "url":"http://example.com/workflows/" - }, - { - "@id": "http://edamontology.org/format_1929", - "@type": "Thing", - "name": "FASTA sequence format" - }, - { - "@id": "http://edamontology.org/format_1982", - "@type": "Thing", - "name": "ClustalW alignment format" - }, - { - "@id": "http://edamontology.org/format_2572", - "@type": "Thing", - "name": "BAM format" - }, - { - "@id": "http://edamontology.org/data_2977", - "@type": "Thing", - "name": "Nucleic acid sequence" - }, - { - "@id": "http://edamontology.org/data_1383", - "@type": "Thing", - "name": "Nucleic acid sequence alignment" - } - ] -} -``` - - -```java - ContextualEntity license = new ContextualEntity.ContextualEntityBuilder() - .addType("CreativeWork") - .setId("https://spdx.org/licenses/CC-BY-NC-SA-4.0") - .addProperty("name", "Creative Commons Attribution Non Commercial Share Alike 4.0 International") - .addProperty("alternateName", "CC-BY-NC-SA-4.0") - .build(); - ContextualEntity knime = new ContextualEntity.ContextualEntityBuilder() - .setId("#knime") - .addType("ProgrammingLanguage") - .addProperty("name", "KNIME Analytics Platform") - .addProperty("alternateName", "KNIME") - .addProperty("url", "https://www.knime.com/whats-new-in-knime-41") - .addProperty("version", "4.1.3") - .build(); - OrganizationEntity workflowHub = new OrganizationEntity.OrganizationEntityBuilder() - .setId("#workflow-hub") - .addProperty("name", "Example Workflow Hub") - .addProperty("url", "http://example.com/workflows/") - .build(); - ContextualEntity fasta = new ContextualEntity.ContextualEntityBuilder() - .setId("http://edamontology.org/format_1929") - .addType("Thing") - .addProperty("name", "FASTA sequence format") - .build(); - ContextualEntity clustalW = new ContextualEntity.ContextualEntityBuilder() - .setId("http://edamontology.org/format_1982") - .addType("Thing") - .addProperty("name", "ClustalW alignment format") - .build(); - ContextualEntity ban = new ContextualEntity.ContextualEntityBuilder() - .setId("http://edamontology.org/format_2572") - .addType("Thing") - .addProperty("name", "BAM format") - .build(); - ContextualEntity nucSec = new ContextualEntity.ContextualEntityBuilder() - .setId("http://edamontology.org/data_2977") - .addType("Thing") - .addProperty("name", "Nucleic acid sequence") - .build(); - ContextualEntity nucAlign = new ContextualEntity.ContextualEntityBuilder() - .setId("http://edamontology.org/data_1383") - .addType("Thing") - .addProperty("name", "Nucleic acid sequence alignment") - .build(); - PersonEntity alice = new PersonEntity.PersonEntityBuilder() - .setId("#alice") - .addProperty("name", "Alice Brown") - .build(); - ContextualEntity requiredParam = new ContextualEntity.ContextualEntityBuilder() - .addType("FormalParameter") - .setId("#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b") - .addProperty("name", "genome_sequence") - .addProperty("valueRequired", true) - .addIdProperty("conformsTo", "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/") - .addIdProperty("additionalType", nucSec) - .addIdProperty("encodingFormat", fasta) - .build(); - ContextualEntity clnParam = new ContextualEntity.ContextualEntityBuilder() - .addType("FormalParameter") - .setId("#6c703fee-6af7-4fdb-a57d-9e8bc4486044") - .addProperty("name", "cleaned_sequence") - .addIdProperty("conformsTo", "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/") - .addIdProperty("additionalType", nucSec) - .addIdProperty("encodingFormat", ban) - .build(); - ContextualEntity alignParam = new ContextualEntity.ContextualEntityBuilder() - .addType("FormalParameter") - .setId("#2f32b861-e43c-401f-8c42-04fd84273bdf") - .addProperty("name", "sequence_alignment") - .addIdProperty("conformsTo", "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/") - .addIdProperty("additionalType", nucAlign) - .addIdProperty("encodingFormat", clustalW) - .build(); - - RoCrate crate = new RoCrate.RoCrateBuilder("Example RO-Crate", "The RO-Crate Root Data Entity", "2020", "https://spdx.org/licenses/CC-BY-NC-SA-4.0") - .addContextualEntity(license) - .addContextualEntity(knime) - .addContextualEntity(workflowHub) - .addContextualEntity(fasta) - .addContextualEntity(clustalW) - .addContextualEntity(ban) - .addContextualEntity(nucSec) - .addContextualEntity(nucAlign) - .addContextualEntity(alice) - .addContextualEntity(requiredParam) - .addContextualEntity(clnParam) - .addContextualEntity(alignParam) - .addDataEntity( - new WorkflowEntity.WorkflowEntityBuilder() - .setId("workflow/alignment.knime") - .setSource(new File("src")) - .addIdProperty("conformsTo", "https://bioschemas.org/profiles/ComputationalWorkflow/0.5-DRAFT-2020_07_21/") - .addProperty("name", "Sequence alignment workflow") - .addIdProperty("programmingLanguage", "#knime") - .addAuthor("#alice") - .addProperty("dateCreated", "2020-05-23") - .setLicense("https://spdx.org/licenses/CC-BY-NC-SA-4.0") - .addInput("#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b") - .addOutput("#6c703fee-6af7-4fdb-a57d-9e8bc4486044") - .addOutput("#2f32b861-e43c-401f-8c42-04fd84273bdf") - .addProperty("url", "http://example.com/workflows/alignment") - .addProperty("version", "0.5.0") - .addIdProperty("sdPublisher", "#workflow-hub") - .build() - - ) - .build(); -``` +We have an [example module with unit tests](src/test/java/edu/kit/datamanager/ro_crate/example/), describing how to generate the official [examples from the specification](https://www.researchobject.org/ro-crate/1.1/). +Specifically, the examples for the specification in version 1.1 are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java index 3c7ca0e7..d699eeae 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java @@ -3,12 +3,10 @@ import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; +import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity; import edu.kit.datamanager.ro_crate.entities.contextual.PersonEntity; import edu.kit.datamanager.ro_crate.entities.contextual.PlaceEntity; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; +import edu.kit.datamanager.ro_crate.entities.data.*; import edu.kit.datamanager.ro_crate.writer.CrateWriter; import org.junit.jupiter.api.Test; @@ -262,4 +260,148 @@ void testWithFileAuthorLocation() { printAndAssertEquals(crate, "/spec-v1.1-example-json-files/file-author-location.json"); } + + /** + * From: + * Example with complete workflow + * (location in repo) + *

+ * This example shows how to connect entities. If there is no specific method like + * {@link DataEntity.DataEntityBuilder#addAuthor(String)} for referencing other + * entities, one can use the more generic + * {@link AbstractEntity.AbstractEntityBuilder#addIdProperty(String, AbstractEntity)} + * or {@link AbstractEntity.AbstractEntityBuilder#addIdProperty(String, String)}. + *

+ * Important Note! If you connect entities, make sure all entities are being + * added to the crate. We currently can't enforce this properly yet. + */ + @Test + void testCompleteWorkflowExample() { + final String licenseId = "https://spdx.org/licenses/CC-BY-NC-SA-4.0"; + ContextualEntity license = new ContextualEntity.ContextualEntityBuilder() + .addType("CreativeWork") + .setId(licenseId) + .addProperty("name", "Creative Commons Attribution Non Commercial Share Alike 4.0 International") + .addProperty("alternateName", "CC-BY-NC-SA-4.0") + .build(); + ContextualEntity knime = new ContextualEntity.ContextualEntityBuilder() + .setId("#knime") + .addType("ComputerLanguage") + .addProperty("name", "KNIME Analytics Platform") + .addProperty("alternateName", "KNIME") + .addProperty("url", "https://www.knime.com/whats-new-in-knime-41") + .addProperty("version", "4.1.3") + .build(); + OrganizationEntity workflowHub = new OrganizationEntity.OrganizationEntityBuilder() + .setId("#workflow-hub") + .addProperty("name", "Example Workflow Hub") + .addProperty("url", "http://example.com/workflows/") + .build(); + ContextualEntity fasta = new ContextualEntity.ContextualEntityBuilder() + .setId("http://edamontology.org/format_1929") + .addType("Thing") + .addProperty("name", "FASTA sequence format") + .build(); + ContextualEntity clustalW = new ContextualEntity.ContextualEntityBuilder() + .setId("http://edamontology.org/format_1982") + .addType("Thing") + .addProperty("name", "ClustalW alignment format") + .build(); + ContextualEntity ban = new ContextualEntity.ContextualEntityBuilder() + .setId("http://edamontology.org/format_2572") + .addType("Thing") + .addProperty("name", "BAM format") + .build(); + ContextualEntity nucSec = new ContextualEntity.ContextualEntityBuilder() + .setId("http://edamontology.org/data_2977") + .addType("Thing") + .addProperty("name", "Nucleic acid sequence") + .build(); + ContextualEntity nucAlign = new ContextualEntity.ContextualEntityBuilder() + .setId("http://edamontology.org/data_1383") + .addType("Thing") + .addProperty("name", "Nucleic acid sequence alignment") + .build(); + PersonEntity alice = new PersonEntity.PersonEntityBuilder() + .setId("#alice") + .addProperty("name", "Alice Brown") + .build(); + ContextualEntity requiredParam = new ContextualEntity.ContextualEntityBuilder() + .addType("FormalParameter") + .setId("#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b") + .addProperty("name", "genome_sequence") + .addProperty("valueRequired", true) + .addIdProperty("conformsTo", "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/") + .addIdProperty("additionalType", nucSec) + .addIdProperty("format", fasta) + .build(); + ContextualEntity clnParam = new ContextualEntity.ContextualEntityBuilder() + .addType("FormalParameter") + .setId("#6c703fee-6af7-4fdb-a57d-9e8bc4486044") + .addProperty("name", "cleaned_sequence") + .addIdProperty("conformsTo", "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/") + .addIdProperty("additionalType", nucSec) + .addIdProperty("encodingFormat", ban) + .build(); + ContextualEntity alignParam = new ContextualEntity.ContextualEntityBuilder() + .addType("FormalParameter") + .setId("#2f32b861-e43c-401f-8c42-04fd84273bdf") + .addProperty("name", "sequence_alignment") + .addIdProperty("conformsTo", "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/") + .addIdProperty("additionalType", nucAlign) + .addIdProperty("encodingFormat", clustalW) + .build(); + + RoCrate crate = new RoCrate.RoCrateBuilder( + "Example RO-Crate", + "The RO-Crate Root Data Entity", + "2020", + licenseId + ) + .setLicense(license) + .addContextualEntity(knime) + .addContextualEntity(workflowHub) + .addContextualEntity(fasta) + .addContextualEntity(clustalW) + .addContextualEntity(ban) + .addContextualEntity(nucSec) + .addContextualEntity(nucAlign) + .addContextualEntity(alice) + .addContextualEntity(requiredParam) + .addContextualEntity(clnParam) + .addContextualEntity(alignParam) + .addDataEntity( + new WorkflowEntity.WorkflowEntityBuilder() + .setId("workflow/alignment.knime") + .setLocation(Paths.get("src")) + .addIdProperty("conformsTo", "https://bioschemas.org/profiles/ComputationalWorkflow/0.5-DRAFT-2020_07_21/") + .addProperty("name", "Sequence alignment workflow") + .addIdProperty("programmingLanguage", "#knime") + // This example does not use the term "author"... + //.addAuthor("#alice") + // instead, it uses "creator": + .addIdProperty("creator", "#alice") + .addProperty("dateCreated", "2020-05-23") + .setLicense(licenseId) + .addInput("#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b") + .addOutput("#6c703fee-6af7-4fdb-a57d-9e8bc4486044") + .addOutput("#2f32b861-e43c-401f-8c42-04fd84273bdf") + .addProperty("url", "http://example.com/workflows/alignment") + .addProperty("version", "0.5.0") + .addIdProperty("sdPublisher", "#workflow-hub") + .build() + ) + .build(); + + // Similar to the previous example, this example from the specification + // spared out some details we now need to remove. + // Here we do not want to remove the license, only the reference to our root data entity. + // This is because (the way we constructed the crate) other entities use the license as well. + crate.getRootDataEntity().removeProperty("license"); + crate.getRootDataEntity().removeProperty("datePublished"); + crate.getRootDataEntity().removeProperty("name"); + crate.getRootDataEntity().removeProperty("description"); + + printAndAssertEquals(crate, "/spec-v1.1-example-json-files/complete-workflow-example.json"); + } } diff --git a/src/test/resources/spec-v1.1-example-json-files/complete-workflow-example.json b/src/test/resources/spec-v1.1-example-json-files/complete-workflow-example.json new file mode 100644 index 00000000..011258f6 --- /dev/null +++ b/src/test/resources/spec-v1.1-example-json-files/complete-workflow-example.json @@ -0,0 +1,113 @@ +{ "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@type": "CreativeWork", + "@id": "ro-crate-metadata.json", + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, + "about": {"@id": "./"} + }, + { + "@id": "./", + "@type": "Dataset", + "hasPart": [ + { "@id": "workflow/alignment.knime" } + ] + }, + { + "@id": "workflow/alignment.knime", + "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], + "conformsTo": + {"@id": "https://bioschemas.org/profiles/ComputationalWorkflow/0.5-DRAFT-2020_07_21/"}, + "name": "Sequence alignment workflow", + "programmingLanguage": {"@id": "#knime"}, + "creator": {"@id": "#alice"}, + "dateCreated": "2020-05-23", + "license": { "@id": "https://spdx.org/licenses/CC-BY-NC-SA-4.0"}, + "input": [ + { "@id": "#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b"} + ], + "output": [ + { "@id": "#6c703fee-6af7-4fdb-a57d-9e8bc4486044"}, + { "@id": "#2f32b861-e43c-401f-8c42-04fd84273bdf"} + ], + "sdPublisher": {"@id": "#workflow-hub"}, + "url": "http://example.com/workflows/alignment", + "version": "0.5.0" + }, + { + "@id": "#36aadbd4-4a2d-4e33-83b4-0cbf6a6a8c5b", + "@type": "FormalParameter", + "conformsTo": {"@id": "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/"}, + "name": "genome_sequence", + "valueRequired": true, + "additionalType": {"@id": "http://edamontology.org/data_2977"}, + "format": {"@id": "http://edamontology.org/format_1929"} + }, + { + "@id": "#6c703fee-6af7-4fdb-a57d-9e8bc4486044", + "@type": "FormalParameter", + "conformsTo": {"@id": "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/"}, + "name": "cleaned_sequence", + "additionalType": {"@id": "http://edamontology.org/data_2977"}, + "encodingFormat": {"@id": "http://edamontology.org/format_2572"} + }, + { + "@id": "#2f32b861-e43c-401f-8c42-04fd84273bdf", + "@type": "FormalParameter", + "conformsTo": {"@id": "https://bioschemas.org/profiles/FormalParameter/0.1-DRAFT-2020_07_21/"}, + "name": "sequence_alignment", + "additionalType": {"@id": "http://edamontology.org/data_1383"}, + "encodingFormat": {"@id": "http://edamontology.org/format_1982"} + }, + { + "@id": "https://spdx.org/licenses/CC-BY-NC-SA-4.0", + "@type": "CreativeWork", + "name": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + "alternateName": "CC-BY-NC-SA-4.0" + }, + { + "@id": "#knime", + "@type": "ComputerLanguage", + "name": "KNIME Analytics Platform", + "alternateName": "KNIME", + "url": "https://www.knime.com/whats-new-in-knime-41", + "version": "4.1.3" + }, + { + "@id": "#alice", + "@type": "Person", + "name": "Alice Brown" + }, + { + "@id": "#workflow-hub", + "@type": "Organization", + "name": "Example Workflow Hub", + "url":"http://example.com/workflows/" + }, + { + "@id": "http://edamontology.org/format_1929", + "@type": "Thing", + "name": "FASTA sequence format" + }, + { + "@id": "http://edamontology.org/format_1982", + "@type": "Thing", + "name": "ClustalW alignment format" + }, + { + "@id": "http://edamontology.org/format_2572", + "@type": "Thing", + "name": "BAM format" + }, + { + "@id": "http://edamontology.org/data_2977", + "@type": "Thing", + "name": "Nucleic acid sequence" + }, + { + "@id": "http://edamontology.org/data_1383", + "@type": "Thing", + "name": "Nucleic acid sequence alignment" + } + ] +} From 3eec23d5567cdc700d84862bb2b17a8bbd744d5d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Sat, 3 May 2025 01:24:53 +0200 Subject: [PATCH 21/79] docs: refactor specification examples section in README into other existing sections --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index edb7c12e..0d1d068b 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ and avoiding crates which do not fully comply to the specification, at the same - [Instructions for your build manager (e.g., Gradle, Maven, etc.)](https://central.sonatype.com/artifact/edu.kit.datamanager/ro-crate-java) - [Quick-Start](#quick-start) -- [Adapting Specification Examples](#adapting-the-specification-examples) - [Related Publications](https://publikationen.bibliothek.kit.edu/publikationslisten/get.php?referencing=all&external_publications=kit&lang=de&format=html&style=kit-3lines-title_b-authors-other&consider_suborganizations=true&order=desc%20year&contributors=%5B%5B%5B%5D%2C%5B%22p20751.105%22%5D%5D%5D&title_contains=crate) ## Build the library / documentation @@ -31,12 +30,16 @@ On Windows, replace `./gradlew` with `gradlew.bat`. ## RO-Crate Specification Compatibility -- ✅ Version 1.1 +- ✅ [Version 1.1](https://www.researchobject.org/ro-crate/1.1/) ([Extracted examples as well-described unit tests/guide](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java)) - 🛠️ Version 1.2-DRAFT - ✅ Reading and writing crates with additional profiles or specifications ([examples for reading](src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java), [examples for writing](src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java)) - ✅ Adding profiles or other specifications to a crate ([examples](src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java)) ## Quick-start + +> Note: We have a [module with well-described unit tests to guide you](src/test/java/edu/kit/datamanager/ro_crate/examples/), describing how to generate official examples extracted from the specification. +> Specifically, the examples for the [specification in version 1.1](https://www.researchobject.org/ro-crate/1.1/) are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). + ### Example for a basic crate from [RO-Crate website](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#ro-crate-metadata-file-descriptor) ```java RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "licenseIdentifier").build(); @@ -176,7 +179,7 @@ RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "li Keep in mind that, if you want to use AutomaticPreview, you have to install ro-crate-html-js via `npm install --global ro-crate-html-js` first. -For StaticPreview, the constuctor is a bit different, such that it looks as follows: +For StaticPreview, the constructor is a bit different, such that it looks as follows: ```java File pathToMainPreviewHtml = new File("localPath"); @@ -193,8 +196,3 @@ Right now, the only implemented way of validating a RO-crate is to use a [JSON-S Validator validator = new Validator(new JsonSchemaValidation("./schema.json")); boolean valid = validator.validate(crate); ``` - -## Adapting the specification examples - -We have an [example module with unit tests](src/test/java/edu/kit/datamanager/ro_crate/example/), describing how to generate the official [examples from the specification](https://www.researchobject.org/ro-crate/1.1/). -Specifically, the examples for the specification in version 1.1 are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). From 61fd863d187498b119294704381e1d4c7ee65b58 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Sat, 3 May 2025 16:23:08 +0200 Subject: [PATCH 22/79] docs: move quickstart to LearnByExampleTest so it is always tested properly --- README.md | 168 +------ .../ro_crate/preview/StaticPreview.java | 3 +- .../ro_crate/examples/LearnByExampleTest.java | 417 ++++++++++++++++++ 3 files changed, 435 insertions(+), 153 deletions(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java diff --git a/README.md b/README.md index 0d1d068b..6d896722 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ and avoiding crates which do not fully comply to the specification, at the same - [Instructions for your build manager (e.g., Gradle, Maven, etc.)](https://central.sonatype.com/artifact/edu.kit.datamanager/ro-crate-java) - [Quick-Start](#quick-start) +- [JavaDoc Documentation](https://javadoc.io/doc/edu.kit.datamanager/ro-crate-java) - [Related Publications](https://publikationen.bibliothek.kit.edu/publikationslisten/get.php?referencing=all&external_publications=kit&lang=de&format=html&style=kit-3lines-title_b-authors-other&consider_suborganizations=true&order=desc%20year&contributors=%5B%5B%5B%5D%2C%5B%22p20751.105%22%5D%5D%5D&title_contains=crate) ## Build the library / documentation @@ -37,162 +38,27 @@ On Windows, replace `./gradlew` with `gradlew.bat`. ## Quick-start -> Note: We have a [module with well-described unit tests to guide you](src/test/java/edu/kit/datamanager/ro_crate/examples/), describing how to generate official examples extracted from the specification. -> Specifically, the examples for the [specification in version 1.1](https://www.researchobject.org/ro-crate/1.1/) are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). +` ro-crate-java` makes use of the builder pattern to guide the user to create a valid RO-Crate, similar to: -### Example for a basic crate from [RO-Crate website](https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#ro-crate-metadata-file-descriptor) ```java -RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "licenseIdentifier").build(); -``` - -### Example adding a File (Data Entity) and a context pair -```java -RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "licenseIdentifier") - .addValuePairToContext("Station", "www.station.com") - .addUrlToContext("contextUrl") +RoCrate myFirstCrate = STARTER_CRATE .addDataEntity( - new FileEntity.FileEntityBuilder() - .setId("survey-responses-2019.csv") - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .build() + new FileEntity.FileEntityBuilder() + .setId("path/within/crate/survey-responses-2019.csv") + .setLocation(Paths.get("path/to/current/location/experiment.csv")) + .addProperty("name", "Survey responses") + .addProperty("contentSize", "26452") + .addProperty("encodingFormat", "text/csv") + .build() ) - .addDataEntity(...) - ... - .addContextualEntity(...) - ... - .build(); -``` - -The library currently comes with three specialized DataEntities: - -1. `DataSetEntity` -2. `FileEntity` (used in the example above) -3. `WorkflowEntity` - -If another type of `DataEntity` is required, the base class `DataEntity` can be used. Example: -```java -new DataEntity.DataEntityBuilder() - .addType("CreativeWork") - .setId("ID") - .addProperty("property from schema.org/Creativework", "value") - .build(); -``` -Note that here you are supposed to add the type of your `DataEntity` because it is not known. - -A `DataEntity` and its subclasses can have a file located on the web. Example: - -Example adding file: -```java -new FileEntity.FileEntityBuilder() - .addContent(URI.create("https://github.com/kit-data-manager/ro-crate-java/issues/5")) - .addProperty("description", "my new file that I added") - .build(); -``` - -A `DataEntity` and its subclasses can have a local file associated with them, -instead of one located on the web (which link is the ID of the data entity). Example: - -Example adding file: -```java -new FileEntity.FileEntityBuilder() - .addContent(Paths.get("file"), "new_file.txt") - .addProperty("description", "my new local file that I added") - .build(); -``` - -### Contextual Entities - -Contextual entities cannot be associated with a file (they are pure metadata). - -To add a contextual entity to a crate you use the function `.addContextualEntity(ContextualEntity entity)`. -Some types of derived/specializes entities are: -1. `OrganizationEntity` -2. `PersonEntity` -3. `PlaceEntity` - -If you need another type of contextual entity, use the base class `ContextualEntity`. - -The library provides a way to automatically create contextual entities from external providers. Currently, support for [ORCID](https://orcid.org/) and [ROR](https://ror.org/) is implemented. Example: -```java -PersonEntity person = ORCIDProvider.getPerson("https://orcid.org/*") -OrganizationEntity organization = RORProvider.getOrganization("https://ror.org/*"); -``` - -### Writing Crate to folder, zip file, or zip stream - -Writing to folder: -```java -RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); -folderRoCrateWriter.save(roCrate, "destinationFolder"); -``` - -Writing to zip file: -```java -RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); -roCrateZipWriter.save(roCrate, "destinationFolder"); -``` - -Writing to zip stream: -```java -RoCrateWriter roCrateZipStreamWriter = new RoCrateWriter(new ZipStreamWriter()); -roCrateZipStreamWriter.save(roCrate, outputStream); -``` - -More writing strategies can be implemented, if required. - -### Reading / importing Crate from folder or zip - -Reading from folder: -```java -RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); -RoCrate res = roCrateFolderReader.readCrate("destinationFolder"); -``` - -Reading from zip file: -```java -RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipReader()); -RoCrate crate = roCrateFolderReader.readCrate("sourceZipFile"); -``` - -Reading from zip stream: -```java -RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipStreamReader()); -RoCrate crate = roCrateFolderReader.readCrate(inputStream); -``` - -### RO-Crate Website (HTML preview file) -ro-crate-java offers tree different kinds of previews: - -* AutomaticPreview: Uses third-party library [ro-crate-html-js](https://www.npmjs.com/package/ro-crate-html-js), which must be installed separately. -* CustomPreview: Pure Java-based preview using an included template processed by the FreeMarker template engine. At the same time, CustomPreview is the fallback for AutomaticPreview if ro-crate-html-js is not installed. -* StaticPreview: Allows to provide a static HTML page (including additional dependencies, e.g., CSS, JS) which is then shipped with the RO-Crate. - -When creating a new RO-Crate using the builder, the default setting is to use CustomPreview. If you want to change this behaviour, thr preview method is set as follows: - -```java -RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "licenseIdentifier") - .setPreview(new AutomaticPreview()) + .addDataEntity(/*...*/) + .addContextualEntity(/*...*/) .build(); ``` -Keep in mind that, if you want to use AutomaticPreview, you have to install ro-crate-html-js via `npm install --global ro-crate-html-js` first. - -For StaticPreview, the constructor is a bit different, such that it looks as follows: +A built or imported crate can of course also be modified afterwards. Take a look at our further documentation: -```java -File pathToMainPreviewHtml = new File("localPath"); -File pathToAdditionalFiles = new File("localFolder"); -RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "licenseIdentifier") - .setPreview(new StaticPreview(pathToMainPreviewHtml, pathToAdditionalFiles)) - .build(); -``` - -### RO-Crate validation (machine-readable crate profiles) -Right now, the only implemented way of validating a RO-crate is to use a [JSON-Schema](https://json-schema.org/) that the crates metadata JSON file should match. JSON-Schema is an established standard and therefore a good choice for a crate profile. Example: - -```java -Validator validator = new Validator(new JsonSchemaValidation("./schema.json")); -boolean valid = validator.validate(crate); -``` +- **There is a well-documented example-driven guide in [LearnByExampleTest.java](src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java) to help you get started.** +- You may also be interested in the examples we extracted from the [specification in version 1.1](https://www.researchobject.org/ro-crate/1.1/), which are available in [ExamplesOfSpecificationV1p1Test.java](src/test/java/edu/kit/datamanager/ro_crate/examples/ExamplesOfSpecificationV1p1Test.java). +- There is a [module with all well-described guiding tests](src/test/java/edu/kit/datamanager/ro_crate/examples/) available. +- The [JavaDoc Documentation](https://javadoc.io/doc/edu.kit.datamanager/ro-crate-java) is also available online. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index 398615c1..5a627e17 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -3,7 +3,6 @@ import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; import java.io.IOException; -import java.util.Optional; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; @@ -13,7 +12,7 @@ /** * This class adds a static preview to the crate, which consists of a * metadataHtml file and a folder containing other files required to render - * metadataHtml. If will be put unchanged to the writer output, i.e., a zip + * metadataHtml. It will be put unchanged to the writer output, i.e., a zip * file, folder, or stream. * * @author jejkal diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java new file mode 100644 index 00000000..aab61632 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java @@ -0,0 +1,417 @@ +package edu.kit.datamanager.ro_crate.examples; + +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; +import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity; +import edu.kit.datamanager.ro_crate.entities.contextual.PersonEntity; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; +import edu.kit.datamanager.ro_crate.externalproviders.organizationprovider.RorProvider; +import edu.kit.datamanager.ro_crate.externalproviders.personprovider.OrcidProvider; +import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.StaticPreview; +import edu.kit.datamanager.ro_crate.reader.CrateReader; +import edu.kit.datamanager.ro_crate.reader.GenericReaderStrategy; +import edu.kit.datamanager.ro_crate.reader.Readers; +import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; +import edu.kit.datamanager.ro_crate.validation.Validator; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; +import edu.kit.datamanager.ro_crate.writer.FolderStrategy; +import edu.kit.datamanager.ro_crate.writer.GenericWriterStrategy; +import edu.kit.datamanager.ro_crate.writer.Writers; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.*; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * This class is meant to be a small example-driven introduction to the ro-crate-java library. + * It is meant to be read from top to bottom. + */ +public class LearnByExampleTest { + + /** + * This creates a valid, empty RO-Crate builder. + */ + static final RoCrate.RoCrateBuilder STARTER_CRATE = new RoCrate.RoCrateBuilder( + "name", + "description", + "2025", + "licenseIdentifier" + ); + + /** + * Calling the `build()` method on the builder creates a valid RO-Crate. + * Run this test to view the STARTER_CRATE JSON in the console. + */ + @Test + void aSimpleCrate() { + RoCrate almostEmptyCrate = STARTER_CRATE.build(); + assertNotNull(almostEmptyCrate); + HelpFunctions.prettyPrintJsonString(almostEmptyCrate.getJsonMetadata()); + } + + /** + * This is how we can add things to a crate. + *

+ * Note that methods starting with `add` can be used multiple times to add more. + * For example, we can add multiple files or multiple contexts. + *

+ * On the other hand, methods starting with `set` will override previous calls. + *

+ * There may be inconsistencies yet, which are tracked here: Issue #242 + */ + @Test + void addingYourFirstEntity() { + RoCrate myFirstCrate = STARTER_CRATE + // We can add new terms to our crate. The terms we can use are called "context". + .addValuePairToContext("Station", "www.station.com") + // We can also add whole contexts to our crate. + .addUrlToContext("contextUrl") + // Let's add a file to our crate. + .addDataEntity( + new FileEntity.FileEntityBuilder() + // For files (or folders, which are DataSetEntities), + // the ID determines the file name in the crate. + .setId("survey-responses-2019.csv") + // This is where we get the file from. The path will not be part of the metadata. + .setLocation(Paths.get("copy/from/this/file-and-rename-it.csv")) + // And now, the remaining metadata. + // Note that "name", "contentSize", and "encodingFormat" + // are already defined in our default context. + .addProperty("name", "Survey responses") + .addProperty("contentSize", "26452") + .addProperty("encodingFormat", "text/csv") + .build() + ) + // We could add more, but let's keep it simple for now. + //.addDataEntity(...) + //.addContextualEntity(...) + //... + .build(); + + assertNotNull(myFirstCrate); + HelpFunctions.prettyPrintJsonString(myFirstCrate.getJsonMetadata()); + } + + /** + * The library currently comes with three specialized DataEntities: + *

+ * 1. `DataSetEntity` + * 2. `FileEntity` (used in the example above) + * 3. `WorkflowEntity` + *

+ * If another type of `DataEntity` is required, + * the base class `DataEntity` can be used. Example: + */ + @Test + void specializingYourFirstEntity() { + RoCrate crate = STARTER_CRATE + .addDataEntity( + // Let's do something custom: + new DataEntity.DataEntityBuilder() + // You need to add the type of your `DataEntity` + // because for DataEntity, there is no default. + .addType("CreativeWork") + .setId("myEntityInstance") + // Now that we are a CreativeWork instance, + // it is fine to use some of its properties. + .addProperty("https://schema.org/award", "Wow-award") + .build() + ) + .build(); + + assertNotNull(crate); + HelpFunctions.prettyPrintJsonString(crate.getJsonMetadata()); + } + + /** + * A `DataEntity` and its subclasses can have a file located on the web. + * In this case, it does not need to reside in a crate's folder. + * This can be useful for large, publicly available files, + * or in order to reuse or share files. + *

+ * Note: Technically, an entity pointing to a file on the web is just an entity + * that uses the URL as an ID. + */ + @Test + void referencingFilesOnTheWeb() { + // Let's say this is the file we would like to point at with an entity. + String lovelyFile = "https://github.com/kit-data-manager/ro-crate-java/issues/5"; + + RoCrate crate = STARTER_CRATE + .addDataEntity( + // Build our entity to point to the file: + new FileEntity.FileEntityBuilder() + // Make it point to an external file. + .setLocation(URI.create(lovelyFile)) + // This would do the same: + .setId(lovelyFile) + // don't forget to add metadata! + .addProperty("description", "my new file that I added") + .build() + ) + .build(); + + assertNotNull(crate); + HelpFunctions.prettyPrintJsonString(crate.getJsonMetadata()); + } + + /** + * A `DataEntity` and its subclasses can have a local file associated with them, + * instead of one located on the web. + * + * @param tempDir We'll use this to create a temporary folder for our crate. + * @throws IOException If the file cannot be created or written to. + */ + @Test + void includingFilesIntoTheCrateFolder(@TempDir Path tempDir) throws IOException { + // Let's say this is the file we would like to point at with an entity. + String lovelyFile = tempDir.resolve("my/experiment.csv").toString(); + { + // (Let's quickly create a dummy file, but the rest will not make use of this knowledge.) + File lovelyFilePointer = new File(lovelyFile); + FileUtils.touch(lovelyFilePointer); + FileUtils.write(lovelyFilePointer, "My great experiment 001", "UTF-8"); + } + + // But in the crate we want it to be + String seriousExperimentFile = "fantastic-experiment/2025-01-01.csv"; + + RoCrate crate = STARTER_CRATE + .addDataEntity( + // Build our entity to point to the file: + new FileEntity.FileEntityBuilder() + // Let's tell the library where to find and copy the file from. + .setLocation(Paths.get(lovelyFile)) + // Let's tell it to adjust the file name and path in the crate. + .setId(seriousExperimentFile) + .addProperty("description", "my new local file that I added") + .build() + ) + .build(); + + assertNotNull(crate); + HelpFunctions.prettyPrintJsonString(crate.getJsonMetadata()); + + // Let's write it to disk and see if the file is there! + // (We'll discuss writing and reading crates later on.) + Path crateFolder = tempDir.resolve("myCrate"); + Writers.newFolderWriter().save(crate, crateFolder.toString()); + assertTrue(crateFolder.resolve(seriousExperimentFile).toFile().exists()); + } + + /** + * Contextual entities cannot be associated with a file: they are pure metadata + * To add a contextual entity to a crate you use the function + * {@link RoCrate.RoCrateBuilder#addContextualEntity(ContextualEntity)}. + *

+ * Some types of derived/specializes entities are: + *

+ * 1. `OrganizationEntity` + * 2. `PersonEntity` + * 3. `PlaceEntity` + *

+ * If you need another type of contextual entity, use the base class + * {@link ContextualEntity}, similar to how we did it in + * {@link #specializingYourFirstEntity()}. + *

+ * The library provides a way to automatically create contextual entities from + * external providers. Currently, support for [ORCID](https://orcid.org/) and + * [ROR](https://ror.org/) is implemented. + * Check the module {@link edu.kit.datamanager.ro_crate.externalproviders} for + * more implementations. + */ + @Test + void addingContextualEntities() { + PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); + OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); + + RoCrate crate = STARTER_CRATE + .addContextualEntity(person) + .addContextualEntity(organization) + .build(); + + assertNotNull(crate); + HelpFunctions.prettyPrintJsonString(crate.getJsonMetadata()); + } + + /** + * RO-Crates are file based, but in your application you may want to create a crate + * on the fly and directly send it somewhere else without storing it on disk. + * This is why we can't only write to a folder or a zip file, but also to a stream + * (containing the zip file). + *

+ * There is a generic interface to implement Writers (and Readers), so even more + * exotic use cases should be possible. The readers work the same way. + *

+ * - {@link GenericWriterStrategy} + * - {@link GenericReaderStrategy} + */ + @Test + void writingAndReadingCrates(@TempDir Path tempDir) throws IOException { + // Ok lets make a small, but not fully boring crate. + PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); + OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); + + RoCrate crate = STARTER_CRATE + .addContextualEntity(person) + .addContextualEntity(organization) + .build(); + + assertNotNull(crate); + HelpFunctions.prettyPrintJsonString(crate.getJsonMetadata()); + + { + // Now, let's write it to a folder. + Path folder = tempDir.resolve("folderCrate"); + Writers.newFolderWriter() + .save(crate, folder.toString()); + // and read it back. + RoCrate read = Readers.newFolderReader() + .readCrate(folder.toAbsolutePath().toString()); + + HelpFunctions.compareTwoCrateJson(crate, read); + } + + { + // Now, let's write it to a zip file. + Path zipFile = tempDir.resolve("zipCrate.zip"); + Writers.newZipPathWriter() + .save(crate, zipFile.toString()); + // and read it back. + RoCrate read = Readers.newZipPathReader() + .readCrate(zipFile.toAbsolutePath().toString()); + + HelpFunctions.compareTwoCrateJson(crate, read); + } + + { + // Now, let's write it to a zip stream. + Path zipStreamFile = tempDir.resolve("zipStreamCrate.zip"); + try (OutputStream outputStream = new FileOutputStream(zipStreamFile.toFile())) { + Writers.newZipStreamWriter().save(crate, outputStream); + } + // and read it back. + try (InputStream inputStream = new FileInputStream(zipStreamFile.toFile())) { + RoCrate read = Readers.newZipStreamReader() + .readCrate(inputStream); + + HelpFunctions.compareTwoCrateJson(crate, read); + } + } + } + + /** + * In {@link #writingAndReadingCrates(Path)} we already saw how to write or read + * a crate. We used the Readers and Writers classes to get the available options. + * But what if you want to write your own reader or writer strategy? + *

+ * Let's see how you can make a reader or writer, manually configuring the strategy. + */ + @Test + void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { + // Ok lets make a small, but not fully boring crate. + PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); + OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); + + RoCrate crate = STARTER_CRATE + .addContextualEntity(person) + .addContextualEntity(organization) + .build(); + + assertNotNull(crate); + HelpFunctions.prettyPrintJsonString(crate.getJsonMetadata()); + + // Now, let's write it to a folder. Note the used strategy could be replaced with your own. + Path folder = tempDir.resolve("folderCrate"); + new CrateWriter<>(new FolderStrategy()) + .save(crate, folder.toString()); + // and read it back. + RoCrate read = new CrateReader<>( + // Note: There are two FolderStrategy implementations, one for reading and one for writing. + // Java is a bit bad with imports, so we use the fully qualified name here. + new edu.kit.datamanager.ro_crate.reader.FolderStrategy() + ) + .readCrate(folder.toAbsolutePath().toString()); + + HelpFunctions.compareTwoCrateJson(crate, read); + } + + /** + * RO-Crate specified there should be a human-readable preview of the crate. + * This is a HTML file that can be opened in a browser. + * ro-crate-java offers three different ways to create this file: + *

+ * - AutomaticPreview: Uses third-party library + * ro-crate-html-js, + * which must be installed separately via `npm install --global ro-crate-html-js`. + *

+ * - CustomPreview: Pure Java-based preview using an included template processed by + * the FreeMarker template engine. At the same time, CustomPreview is the fallback + * for AutomaticPreview if ro-crate-html-js is not installed. + *

+ * - StaticPreview: Allows to provide a static HTML page (including additional + * dependencies, e.g., CSS, JS) which is then shipped with the RO-Crate. + *

+ * When creating a new RO-Crate using the builder, the default setting is to use + * CustomPreview. This example shows you how to change it. + */ + @Test + void humanReadableContent() { + RoCrate crate = STARTER_CRATE + .setPreview(new AutomaticPreview()) + .build(); + + assertNotNull(crate); + } + + /** + * A static preview means you'll just add your own HTML file to the crate. + * Therefore, the constructor is a bit more complicated. + */ + @Test + void staticPreview(@TempDir Path tempDir) { + File mainPreviewHtml = tempDir.resolve("mainPreview.html").toFile(); + File additionalFilesDirectory = tempDir.resolve("additionalFiles").toFile(); + + RoCrate crate = STARTER_CRATE + .setPreview(new StaticPreview(mainPreviewHtml, additionalFilesDirectory)) + .build(); + + assertNotNull(crate); + } + + /** + * Crates can be validated. + * Right now, the only implemented way of validating a RO-crate is to use a + * [JSON-Schema](https://json-schema.org/) that the crate's metadata JSON file should + * match. JSON-Schema is an established standard and therefore a good choice for a + * crate profile. This example shows how to use it. + *

+ * Note: If you happen to implement your own validator anyway, please consider + * contributing your code! + */ + @Test + void validation() { + // Let's find a schema file in the resources folder. + URL schemaUrl = Objects.requireNonNull(this.getClass().getResource("/crates/validation/workflowschema.json")); + String schemaPath = schemaUrl.getPath(); + + // This crate for sure is not a workflow, so validation will fail. + RoCrate crate = STARTER_CRATE.build(); + + // And now do the validation. + Validator validator = new Validator(new JsonSchemaValidation(schemaPath)); + assertFalse(validator.validate(crate)); + } +} From cdecbed8f6db4bbca42705209a73ffdb3bcd22da Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 12:40:54 +0200 Subject: [PATCH 23/79] test: add parameterized test for reading ELN crates from URLs --- .../ro_crate/reader/ZipReaderTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 3611c834..25babd81 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -2,14 +2,52 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.writer.Writers; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.net.URL; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; class ZipReaderTest extends CrateReaderTest { + /** + * ELN Crates are zip files not fully compatible with the Ro-Crate standard + * in the sense that they must contain a single subfolder in the zip file + * which then contain a crate as specified by the Ro-Crate standard. + *

+ * Here we test if we can read them using out ZipReader. + * + * @see + */ + @ParameterizedTest + @ValueSource(strings = { + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/AI4Green/Export%20workbook-2024-08-27-export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/OpenSemanticLab/MinimalExample.osl.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/PASTA/PASTA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/RSpace/RSpace-2023-12-08-14-44-xml-SELECTION-c0bEtpHcnNe-HA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/SampleDB/sampledb_export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/elabftw/export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" + }) + void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { + // Download the ELN file + URL url = new URL(urlStr); + Path elnFile = tmp.resolve("downloaded.eln"); + FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); + assertTrue(elnFile.toFile().exists()); + + // Read the crate from the downloaded file + Crate read = this.readCrate(elnFile); + assertNotNull(read); + assertFalse(read.getAllDataEntities().isEmpty()); + } + @Override protected void saveCrate(Crate crate, Path target) { Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); From c7e0afceebcec71d133b65fa7c1f8fc8ac802b0b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 13:41:44 +0200 Subject: [PATCH 24/79] refactor: enable readers to implement subsets of tests if they make sense for them. --- ...eReaderTest.java => CommonReaderTest.java} | 60 +++++-------------- .../ro_crate/reader/ElnFileFormatTest.java | 53 ++++++++++++++++ .../ro_crate/reader/FolderReaderTest.java | 12 ++-- .../reader/TestableReaderStrategy.java | 50 ++++++++++++++++ .../ro_crate/reader/ZipReaderTest.java | 52 +++------------- .../ro_crate/reader/ZipStreamReaderTest.java | 13 ++-- 6 files changed, 139 insertions(+), 101 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/reader/{CrateReaderTest.java => CommonReaderTest.java} (78%) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java similarity index 78% rename from src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java index bcf8948d..6cdc6fa1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java @@ -27,9 +27,13 @@ * This parameter is only required to satisfy the generic reader strategy. * @param the type of the reader strategy */ -abstract class CrateReaderTest> { - - protected static RoCrate.RoCrateBuilder newBaseCrate() { +interface CommonReaderTest< + SOURCE_T, + READER_STRATEGY extends GenericReaderStrategy + > + extends TestableReaderStrategy +{ + static RoCrate.RoCrateBuilder newBaseCrate() { return new RoCrate.RoCrateBuilder( "minimal", "minimal RO_crate", @@ -38,7 +42,7 @@ protected static RoCrate.RoCrateBuilder newBaseCrate() { ); } - protected static FileEntity newDataEntity(Path filePath) throws IllegalArgumentException { + static FileEntity newDataEntity(Path filePath) throws IllegalArgumentException { return new FileEntity.FileEntityBuilder() .setLocationWithExceptions(filePath) .setId(filePath.toFile().getName()) @@ -48,44 +52,8 @@ protected static FileEntity newDataEntity(Path filePath) throws IllegalArgumentE .build(); } - /** - * Saves the crate with the writer fitting to the reader of {@link #readCrate(Path)}. - * - * @param crate the crate to save - * @param target the target path to the save location - * @throws IOException if an error occurs while saving the crate - */ - abstract protected void saveCrate(Crate crate, Path target) throws IOException; - - /** - * Reads the crate with the reader fitting to the writer of {@link #saveCrate(Crate, Path)}. - * @param source the source path to the crate - * @return the read crate - * @throws IOException if an error occurs while reading the crate - */ - abstract protected Crate readCrate(Path source) throws IOException; - - /** - * Creates a new reader strategy with a non-default temporary directory (if supported, default otherwise). - * - * @param tmpDirectory the temporary directory to use - * @param useUuidSubfolder whether to create a UUID subfolder under the temporary directory - * @return a new reader strategy - */ - abstract protected READER_STRATEGY newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder); - - /** - * Reads the crate using the provided reader strategy. - * - * @param strategy the reader strategy to use - * @param source the source path to the crate - * @return the read crate - * @throws IOException if an error occurs while reading the crate - */ - abstract protected Crate readCrate(READER_STRATEGY strategy, Path source) throws IOException; - @Test - void testReadingBasicCrate(@TempDir Path temp) throws IOException { + default void testReadingBasicCrate(@TempDir Path temp) throws IOException { RoCrate roCrate = newBaseCrate().build(); Path zipPath = temp.resolve("result.zip"); @@ -95,7 +63,7 @@ void testReadingBasicCrate(@TempDir Path temp) throws IOException { } @Test - void testWithFile(@TempDir Path temp) throws IOException { + default void testWithFile(@TempDir Path temp) throws IOException { Path csvPath = temp.resolve("survey-responses-2019.csv"); FileUtils.touch(csvPath.toFile()); FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); @@ -113,7 +81,7 @@ void testWithFile(@TempDir Path temp) throws IOException { } @Test - void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { + default void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { // This URL will be encoded because of whitespaces Path csvPath = temp.resolve("survey responses 2019.csv"); FileUtils.touch(csvPath.toFile()); @@ -140,7 +108,7 @@ void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { } @Test - void TestWithFileWithLocation(@TempDir Path temp) throws IOException { + default void TestWithFileWithLocation(@TempDir Path temp) throws IOException { Path csvPath = temp.resolve("survey-responses-2019.csv"); FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); RoCrate rawCrate = newBaseCrate() @@ -168,7 +136,7 @@ void TestWithFileWithLocation(@TempDir Path temp) throws IOException { } @Test - void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { + default void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { Path csvPath = temp.resolve("file.csv"); FileUtils.writeStringToFile(csvPath.toFile(), "fakecsv.1", Charset.defaultCharset()); RoCrate rawCrate = newBaseCrate() @@ -206,7 +174,7 @@ void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { } @Test - void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { + default void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { RoCrate rawCrate = newBaseCrate().build(); // Write to zip file diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java new file mode 100644 index 00000000..5aa01b80 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -0,0 +1,53 @@ +package edu.kit.datamanager.ro_crate.reader; + +import edu.kit.datamanager.ro_crate.Crate; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +public interface ElnFileFormatTest< + SOURCE_T, + READER_STRATEGY extends GenericReaderStrategy + > + extends TestableReaderStrategy +{ + /** + * ELN Crates are zip files not fully compatible with the Ro-Crate standard + * in the sense that they must contain a single subfolder in the zip file + * which then contain a crate as specified by the Ro-Crate standard. + *

+ * Here we test if we can read them using out ZipReader. + * + * @see + */ + @ParameterizedTest + @ValueSource(strings = { + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/AI4Green/Export%20workbook-2024-08-27-export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/OpenSemanticLab/MinimalExample.osl.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/PASTA/PASTA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/RSpace/RSpace-2023-12-08-14-44-xml-SELECTION-c0bEtpHcnNe-HA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/SampleDB/sampledb_export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/elabftw/export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" + }) + default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { + // Download the ELN file + URL url = new URL(urlStr); + Path elnFile = tmp.resolve("downloaded.eln"); + FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); + assertTrue(elnFile.toFile().exists()); + + // Read the crate from the downloaded file + Crate read = this.readCrate(elnFile); + assertNotNull(read); + assertFalse(read.getAllDataEntities().isEmpty()); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index 21850edd..a7143862 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -17,21 +17,21 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderReaderTest extends CrateReaderTest { - +class FolderReaderTest implements CommonReaderTest +{ @Override - protected void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) { Writers.newFolderWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isDirectory()); } @Override - protected Crate readCrate(Path source) throws IOException { + public Crate readCrate(Path source) throws IOException { return Readers.newFolderReader().readCrate(source.toAbsolutePath().toString()); } @Override - protected FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { // This strategy does not support a non-default temporary directory // and will always use the default one. // It also has no state we could make assertions on. @@ -39,7 +39,7 @@ protected FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean use } @Override - protected Crate readCrate(FolderStrategy strategy, Path source) throws IOException { + public Crate readCrate(FolderStrategy strategy, Path source) throws IOException { return new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java new file mode 100644 index 00000000..c8d4b72a --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java @@ -0,0 +1,50 @@ +package edu.kit.datamanager.ro_crate.reader; + +import edu.kit.datamanager.ro_crate.Crate; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Base Interface for methods required to test all reader strategies. + * + * @param the source type the strategy reads from. + * @param the type of the reader strategy. + */ +interface TestableReaderStrategy> { + /** + * Saves the crate with the writer fitting to the reader of {@link #readCrate(Path)}. + * + * @param crate the crate to save + * @param target the target path to the save location + * @throws IOException if an error occurs while saving the crate + */ + void saveCrate(Crate crate, Path target) throws IOException; + + /** + * Reads the crate with the reader fitting to the writer of {@link #saveCrate(Crate, Path)}. + * @param source the source path to the crate + * @return the read crate + * @throws IOException if an error occurs while reading the crate + */ + Crate readCrate(Path source) throws IOException; + + /** + * Creates a new reader strategy with a non-default temporary directory (if supported, default otherwise). + * + * @param tmpDirectory the temporary directory to use + * @param useUuidSubfolder whether to create a UUID subfolder under the temporary directory + * @return a new reader strategy + */ + READER_STRATEGY newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder); + + /** + * Reads the crate using the provided reader strategy. + * + * @param strategy the reader strategy to use + * @param source the source path to the crate + * @return the read crate + * @throws IOException if an error occurs while reading the crate + */ + Crate readCrate(READER_STRATEGY strategy, Path source) throws IOException; +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 25babd81..a4616f8e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -2,65 +2,29 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.writer.Writers; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; -import java.net.URL; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -class ZipReaderTest extends CrateReaderTest { - - /** - * ELN Crates are zip files not fully compatible with the Ro-Crate standard - * in the sense that they must contain a single subfolder in the zip file - * which then contain a crate as specified by the Ro-Crate standard. - *

- * Here we test if we can read them using out ZipReader. - * - * @see - */ - @ParameterizedTest - @ValueSource(strings = { - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/AI4Green/Export%20workbook-2024-08-27-export.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/OpenSemanticLab/MinimalExample.osl.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/PASTA/PASTA.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/RSpace/RSpace-2023-12-08-14-44-xml-SELECTION-c0bEtpHcnNe-HA.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/SampleDB/sampledb_export.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/elabftw/export.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" - }) - void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { - // Download the ELN file - URL url = new URL(urlStr); - Path elnFile = tmp.resolve("downloaded.eln"); - FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); - assertTrue(elnFile.toFile().exists()); - - // Read the crate from the downloaded file - Crate read = this.readCrate(elnFile); - assertNotNull(read); - assertFalse(read.getAllDataEntities().isEmpty()); - } - +class ZipReaderTest implements + CommonReaderTest, + ElnFileFormatTest +{ @Override - protected void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) { Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isFile()); } @Override - protected Crate readCrate(Path source) throws IOException { + public Crate readCrate(Path source) throws IOException { return Readers.newZipPathReader().readCrate(source.toAbsolutePath().toString()); } @Override - protected ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { ZipStrategy strategy = new ZipStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { @@ -73,7 +37,7 @@ protected ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUui } @Override - protected Crate readCrate(ZipStrategy strategy, Path source) throws IOException { + public Crate readCrate(ZipStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); assertTrue(strategy.isExtracted()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index 33436c6c..17b37247 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -9,20 +9,23 @@ import static org.junit.jupiter.api.Assertions.*; -class ZipStreamReaderTest extends CrateReaderTest { +class ZipStreamReaderTest implements + CommonReaderTest, + ElnFileFormatTest +{ @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); assertTrue(target.toFile().isFile()); } @Override - protected Crate readCrate(Path source) throws IOException { + public Crate readCrate(Path source) throws IOException { return Readers.newZipStreamReader().readCrate(new FileInputStream(source.toFile())); } @Override - protected ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { ZipStreamStrategy strategy = new ZipStreamStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { @@ -35,7 +38,7 @@ protected ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean } @Override - protected Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { + public Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(new FileInputStream(source.toFile())); assertTrue(strategy.isExtracted()); From 2e0df3510c13e3e42c25201e4d6181a80401a736 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 14:46:49 +0200 Subject: [PATCH 25/79] feat: reading zip files now searches for a crate in subdirectories. Not recursive and limited to 50 folders to avoid load vulnerabilities. --- .../entities/contextual/JsonDescriptor.java | 4 ++-- .../datamanager/ro_crate/reader/ZipStrategy.java | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java index 8bb91294..18acd4c6 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java @@ -13,8 +13,8 @@ public class JsonDescriptor extends ContextualEntity { - private static final String CONFORMS_TO = "conformsTo"; - protected static final String ID = "ro-crate-metadata.json"; + protected static final String CONFORMS_TO = "conformsTo"; + public static final String ID = "ro-crate-metadata.json"; /** * Returns a JsonDescriptor with the conformsTo value set to the latest stable diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java index 0d6381a6..26118841 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; @@ -106,7 +107,17 @@ public ObjectNode readMetadataJson(String location) { } ObjectMapper objectMapper = MyObjectMapper.getMapper(); - File jsonMetadata = temporaryFolder.resolve("ro-crate-metadata.json").toFile(); + File jsonMetadata = this.temporaryFolder.resolve(JsonDescriptor.ID).toFile(); + if (!jsonMetadata.isFile()) { + // Try to find the metadata file in subdirectories + File firstSubdir = FileUtils.listFilesAndDirs(temporaryFolder.toFile(), null, null) + .stream() + .limit(50) + .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); + jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); + } try { return objectMapper.readTree(jsonMetadata).deepCopy(); From ccc0fe841e560b1008dfd6972e53171149aa6719 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:03:20 +0200 Subject: [PATCH 26/79] chore: fix deprecation warning in ElnFileFormatTest and enable better deprecation linter errors --- build.gradle | 5 +++++ .../kit/datamanager/ro_crate/reader/ElnFileFormatTest.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7a9d9647..f7fcd613 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,11 @@ dependencies { implementation("org.freemarker:freemarker:2.3.34") } +// enable -Xlint:deprecation +tasks.withType(JavaCompile).configureEach { + options.compilerArgs << "-Xlint:deprecation" +} + logging.captureStandardOutput LogLevel.INFO def signingTasks = tasks.withType(Sign) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java index 5aa01b80..3e06fc7f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -7,6 +7,7 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.file.Path; @@ -40,7 +41,7 @@ public interface ElnFileFormatTest< }) default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { // Download the ELN file - URL url = new URL(urlStr); + URL url = URI.create(urlStr).toURL(); Path elnFile = tmp.resolve("downloaded.eln"); FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); assertTrue(elnFile.toFile().exists()); From 0434a737e4bb8651c2403277e12149b2c2fc5635 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:04:21 +0200 Subject: [PATCH 27/79] fix: improve error reporting for JSON schema validation --- .../ro_crate/entities/validation/JsonSchemaValidation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java index 067f5e2e..18e9624a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java @@ -60,6 +60,7 @@ public boolean validateEntity(JsonNode entity) { Set errors = this.entitySchema.validate(entity); if (errors.size() != 0) { System.err.println("This entity does not comply to the basic RO-Crate entity structure."); + errors.forEach(error -> System.err.println(error.getMessage())); return false; } return true; From 966255fb0788c12242bc390c20a29b884974e124 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:52:16 +0200 Subject: [PATCH 28/79] refactor: replace deprecated setAll method with setAllIfValid and setAllUnsafe in entity builders --- .../ro_crate/entities/AbstractEntity.java | 46 +++++++++++++++++-- .../dataentities/ImportFromZenodo.java | 4 +- .../personprovider/OrcidProvider.java | 2 +- .../ro_crate/reader/CrateReader.java | 8 ++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 8699fd5b..d3afdc6c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -558,12 +558,19 @@ public T addIdFromCollectionOfEntities(String name, Collection e } /** - * This sets everything from a json object to the property. Can be - * useful when the entity is already available somewhere. + * Deprecated. Equivalent to {@link #setAllIfValid(ObjectNode)}. * * @param properties the Json representing all the properties. - * @return the generic builder. + * @return the generic builder, either including all given properties + * * or unchanged. + * + * @deprecated To enforce the user know what this method does, + * we want the user to use one of the more explicitly named + * methods {@link #setAllIfValid(ObjectNode)} or + * {@link #setAllUnsafe(ObjectNode)}. + * @see #setAllIfValid(ObjectNode) */ + @Deprecated(since = "2.1.0", forRemoval = true) public T setAll(ObjectNode properties) { if (AbstractEntity.entityValidation.entityValidation(properties)) { this.properties = properties; @@ -572,6 +579,39 @@ public T setAll(ObjectNode properties) { return self(); } + /** + * This sets everything from a json object to the property, + * if the result is valid. Otherwise, it will do nothing. + * + * @param properties the Json representing all the properties. + * @return the generic builder, either including all given properties + * or unchanged. + */ + public T setAllIfValid(ObjectNode properties) { + if (AbstractEntity.entityValidation.entityValidation(properties)) { + this.properties = properties; + this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties)); + } + return self(); + } + + /** + * This sets everything from a json object to the property. Can be + * useful when the entity is already available somewhere. + *

+ * Errors on validation are printed, but everything will be added. + * + * @param properties the Json representing all the properties. + * @return the generic builder with all properties added. + */ + public T setAllUnsafe(ObjectNode properties) { + // This will currently only print errors. + AbstractEntity.entityValidation.entityValidation(properties); + this.properties = properties; + this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties)); + return self(); + } + public abstract T self(); public abstract AbstractEntity build(); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java index 1be5607b..4a99ee8a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java @@ -112,12 +112,12 @@ private static void addToCrateFromZotero(String url, Crate crate) { for (var entity : graph) { if (entity.get("@id").asText().equals(mainId)) { var dataEntity = new DataEntity.DataEntityBuilder() - .setAll((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addDataEntity(dataEntity); } else { // here we have to think of a way to differentiate between data and contextual entities. var contextualEntity = new ContextualEntity.ContextualEntityBuilder() - .setAll((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addContextualEntity(contextualEntity); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java index 8e079af9..17207993 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java @@ -72,7 +72,7 @@ public static PersonEntity getPerson(String url) { node.set(element.getKey(), element.getValue()); } } - return new PersonEntity.PersonEntityBuilder().setAll(node).build(); + return new PersonEntity.PersonEntityBuilder().setAllUnsafe(node).build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); logger.error(errorMessage); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 5acb575b..ba2c2ae8 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -119,7 +119,7 @@ private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet { @@ -133,7 +133,7 @@ private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet extractHasPartIds(ObjectNode root) { private void setCrateDescriptor(RoCrate crate, JsonNode descriptor) { ContextualEntity descriptorEntity = new ContextualEntity.ContextualEntityBuilder() - .setAll(descriptor.deepCopy()) + .setAllUnsafe(descriptor.deepCopy()) .build(); crate.setJsonDescriptor(descriptorEntity); } From d0c1103f9cd6c5387d8355015b332e77fba7a335 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:52:45 +0200 Subject: [PATCH 29/79] refactor: use try-with-resources for CloseableHttpClient in RoCrateMetadataContext --- .../datamanager/ro_crate/context/RoCrateMetadataContext.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java index 731a4c1a..9eac5a3f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java @@ -174,10 +174,9 @@ public void addToContextFromUrl(String url) { } } if (jsonNode == null) { - CloseableHttpClient httpclient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(url); CloseableHttpResponse response; - try { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { response = httpclient.execute(httpGet); jsonNode = objectMapper.readValue(response.getEntity().getContent(), JsonNode.class); From cdb897c0a3c2d536a0504b9d0bb4c5539666b7ca Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:53:30 +0200 Subject: [PATCH 30/79] fix: correct typos and improve formatting in documentation and error messages --- .../ro_crate/context/RoCrateMetadataContext.java | 9 +++++---- .../ro_crate/entities/contextual/JsonDescriptor.java | 2 +- .../edu/kit/datamanager/ro_crate/reader/CrateReader.java | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java index 9eac5a3f..cce6a7e9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java @@ -113,9 +113,10 @@ public boolean checkEntity(AbstractEntity entity) { node.remove("@id"); node.remove("@type"); - Set types = objectMapper.convertValue(entity.getProperties().get("@type"), - new TypeReference<>() { - }); + Set types = objectMapper.convertValue( + entity.getProperties().path("@type"), + new TypeReference<>() {} + ); // check if the items in the array of types are present in the context for (String s : types) { // special cases: @@ -181,7 +182,7 @@ public void addToContextFromUrl(String url) { jsonNode = objectMapper.readValue(response.getEntity().getContent(), JsonNode.class); } catch (IOException e) { - System.err.println(String.format("Cannot get context from url %s", url)); + System.err.printf("Cannot get context from url %s%n", url); return; } if (url.equals(DEFAULT_CONTEXT)) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java index 18acd4c6..88aaf89e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java @@ -39,7 +39,7 @@ private JsonDescriptor(ContextualEntityBuilder builder) { /** * Builder for the JsonDescriptor. - * + *

* Defaults to the latest stable crate version and no other conformsTo values. */ public static final class Builder { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index ba2c2ae8..6586f1ed 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -29,7 +29,7 @@ * The constructor takes a strategy to support different ways of importing the * crates. (from zip, folder, etc.). *

- * The reader consideres "hasPart" and "isPartOf" properties and considers all + * The reader considers "hasPart" and "isPartOf" properties and considers all * entities (in-)directly connected to the root entity ("./") as DataEntities. * * @param the type of the location parameter From 160ba9730c0edd3fb86a75c5b4a0bbf77e5aaf57 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 17:54:08 +0200 Subject: [PATCH 31/79] docs: enhance documentation and add tests for setAllIfValid and setAllUnsafe methods --- .../ro_crate/entities/AbstractEntity.java | 8 +- .../contextual/ContextualEntityTest.java | 105 +++++++++++++++++- 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index d3afdc6c..3ec7bb9f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -567,7 +567,7 @@ public T addIdFromCollectionOfEntities(String name, Collection e * @deprecated To enforce the user know what this method does, * we want the user to use one of the more explicitly named * methods {@link #setAllIfValid(ObjectNode)} or - * {@link #setAllUnsafe(ObjectNode)}. + * {@link #setAllIfValid(ObjectNode)}. * @see #setAllIfValid(ObjectNode) */ @Deprecated(since = "2.1.0", forRemoval = true) @@ -582,6 +582,11 @@ public T setAll(ObjectNode properties) { /** * This sets everything from a json object to the property, * if the result is valid. Otherwise, it will do nothing. + *

+ * Valid means here that the json object needs to be flat as specified + * in the RO-Crate specification. In principle, this means that + * primitives and objects referencing an ID are allowed, + * as well as arrays of these. * * @param properties the Json representing all the properties. * @return the generic builder, either including all given properties @@ -600,6 +605,7 @@ public T setAllIfValid(ObjectNode properties) { * useful when the entity is already available somewhere. *

* Errors on validation are printed, but everything will be added. + * For more about validation, see {@link #setAllIfValid(ObjectNode)}. * * @param properties the Json representing all the properties. * @return the generic builder with all properties added. diff --git a/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java b/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java index 575c941e..7bc868f3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java @@ -1,13 +1,16 @@ package edu.kit.datamanager.ro_crate.entities.contextual; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.io.IOException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + /** * @author Nikola Tzotchev on 5.2.2022 г. * @version 1 @@ -45,4 +48,102 @@ void testSerialization() throws IOException { assertTrue(place.getLinkedTo().contains(geo.getId())); HelpFunctions.compareEntityWithFile(place, "/json/entities/contextual/place.json"); } + + @Test + void testAddAllValidCase() throws JsonProcessingException { + ContextualEntity first = new ContextualEntity.ContextualEntityBuilder() + .setId("#b4168a98-8534-4c6d-a568-64a55157b656") + .addType("GeoCoordinates") + .addProperty("latitude", "-33.7152") + .addProperty("longitude", "150.30119") + .addProperty("name", "Latitude: -33.7152 Longitude: 150.30119") + .build(); + + String allProperties = """ + { + "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", + "@type": "GeoCoordinates", + "latitude": "-33.7152", + "longitude": "150.30119", + "name": "Latitude: -33.7152 Longitude: 150.30119" + } + """; + + ObjectNode properties = MyObjectMapper.getMapper() + .readValue(allProperties, ObjectNode.class); + ContextualEntity second = new ContextualEntity.ContextualEntityBuilder() + .setAllIfValid(properties) + .build(); + assertEquals(second.getProperties(), first.getProperties()); + } + + @Test + void testAddAllInvalidCase() throws JsonProcessingException { + ContextualEntity first = new ContextualEntity.ContextualEntityBuilder() + .setId("#b4168a98-8534-4c6d-a568-64a55157b656") + .addType("GeoCoordinates") + .addProperty("latitude", "-33.7152") + .addProperty("longitude", "150.30119") + .addProperty("name", "Latitude: -33.7152 Longitude: 150.30119") + .build(); + + String allProperties = """ + { + "wrong property": {"any": "value"}, + "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", + "@type": "GeoCoordinates", + "latitude": "-33.7152", + "longitude": "150.30119", + "name": "Latitude: -33.7152 Longitude: 150.30119" + } + """; + + ObjectNode properties = MyObjectMapper.getMapper() + .readValue(allProperties, ObjectNode.class); + ContextualEntity second = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .setAllIfValid(properties) + .build(); + assertNotEquals(second.getProperties(), first.getProperties()); + ObjectNode empty = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .build() + .getProperties(); + assertEquals(empty, second.getProperties()); + } + + @Test + void testAddAllUnsafeDoesInvalidCase() throws JsonProcessingException { + ContextualEntity first = new ContextualEntity.ContextualEntityBuilder() + .setId("#b4168a98-8534-4c6d-a568-64a55157b656") + .addType("GeoCoordinates") + .addProperty("latitude", "-33.7152") + .addProperty("longitude", "150.30119") + .addProperty("name", "Latitude: -33.7152 Longitude: 150.30119") + .build(); + + String allProperties = """ + { + "wrong property": {"any": "value"}, + "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", + "@type": "GeoCoordinates", + "latitude": "-33.7152", + "longitude": "150.30119", + "name": "Latitude: -33.7152 Longitude: 150.30119" + } + """; + + ObjectNode properties = MyObjectMapper.getMapper() + .readValue(allProperties, ObjectNode.class); + ContextualEntity second = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .setAllUnsafe(properties) + .build(); + assertNotEquals(second.getProperties(), first.getProperties()); + ObjectNode empty = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .build() + .getProperties(); + assertNotEquals(empty, second.getProperties()); + } } From 86654f65fbb5dcaf1f6df75ace43524c93331ed4 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 17:58:09 +0200 Subject: [PATCH 32/79] fix: invalid entities from our providers should be logged and not written We only want to add it anyway when reading, but avoid when reading a crate. --- .../externalproviders/dataentities/ImportFromZenodo.java | 4 ++-- .../externalproviders/personprovider/OrcidProvider.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java index 4a99ee8a..2cc6680a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java @@ -112,12 +112,12 @@ private static void addToCrateFromZotero(String url, Crate crate) { for (var entity : graph) { if (entity.get("@id").asText().equals(mainId)) { var dataEntity = new DataEntity.DataEntityBuilder() - .setAllUnsafe((ObjectNode) entity).build(); + .setAllIfValid((ObjectNode) entity).build(); crate.addDataEntity(dataEntity); } else { // here we have to think of a way to differentiate between data and contextual entities. var contextualEntity = new ContextualEntity.ContextualEntityBuilder() - .setAllUnsafe((ObjectNode) entity).build(); + .setAllIfValid((ObjectNode) entity).build(); crate.addContextualEntity(contextualEntity); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java index 17207993..dd0b802b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java @@ -72,7 +72,7 @@ public static PersonEntity getPerson(String url) { node.set(element.getKey(), element.getValue()); } } - return new PersonEntity.PersonEntityBuilder().setAllUnsafe(node).build(); + return new PersonEntity.PersonEntityBuilder().setAllIfValid(node).build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); logger.error(errorMessage); From 7e4a30f3c6dc4abac4970011c16d5e2a51a7d2bc Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 16:02:36 +0200 Subject: [PATCH 33/79] refactor: optimize file search in ZipStrategy by using directory filter --- .../ro_crate/reader/ZipStrategy.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java index 26118841..74622b2f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -6,6 +6,7 @@ import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; import java.io.File; import java.io.IOException; @@ -110,12 +111,16 @@ public ObjectNode readMetadataJson(String location) { File jsonMetadata = this.temporaryFolder.resolve(JsonDescriptor.ID).toFile(); if (!jsonMetadata.isFile()) { // Try to find the metadata file in subdirectories - File firstSubdir = FileUtils.listFilesAndDirs(temporaryFolder.toFile(), null, null) - .stream() - .limit(50) - .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); + File firstSubdir = FileUtils.listFilesAndDirs( + temporaryFolder.toFile(), + FileFilterUtils.directoryFileFilter(), + null // not recursive + ) + .stream() + .limit(50) + .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); } From 49f1b2c64980ad5d3a384632dd315263a519ae69 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 16:02:56 +0200 Subject: [PATCH 34/79] fix: improve metadata file resolution in ZipStreamStrategy to search subdirectories --- .../ro_crate/reader/ZipStreamStrategy.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index cb4f53af..be21f6d4 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import java.io.File; import java.io.FileOutputStream; @@ -13,6 +14,7 @@ import net.lingala.zip4j.io.inputstream.ZipInputStream; import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +132,21 @@ public ObjectNode readMetadataJson(InputStream stream) { } ObjectMapper objectMapper = MyObjectMapper.getMapper(); - File jsonMetadata = temporaryFolder.resolve("ro-crate-metadata.json").toFile(); + File jsonMetadata = temporaryFolder.resolve(JsonDescriptor.ID).toFile(); + if (!jsonMetadata.isFile()) { + // Try to find the metadata file in subdirectories + File firstSubdir = FileUtils.listFilesAndDirs( + temporaryFolder.toFile(), + FileFilterUtils.directoryFileFilter(), + null + ) + .stream() + .limit(50) + .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); + jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); + } try { return objectMapper.readTree(jsonMetadata).deepCopy(); From 6f1948c5228f9a9a346503656f30f2d6fc2ae250 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 16:05:15 +0200 Subject: [PATCH 35/79] fix: handle directory entries in ZipStreamStrategy to ensure proper extraction --- .../kit/datamanager/ro_crate/reader/ZipStreamStrategy.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index be21f6d4..03286a29 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -110,6 +110,11 @@ private void readCrate(InputStream stream) { if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { throw new IOException("Entry is outside of target directory: " + fileName); } + if (localFileHeader.isDirectory()) { + FileUtils.forceMkdir(extractedFile); + continue; + } + FileUtils.forceMkdir(extractedFile.getParentFile()); try (OutputStream outputStream = new FileOutputStream(extractedFile)) { while ((readLen = zipInputStream.read(readBuffer)) != -1) { outputStream.write(readBuffer, 0, readLen); From 5dd5421d77fd165fa316084d1bfd7cc0e7698af4 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 17:10:56 +0200 Subject: [PATCH 36/79] fix: integrate Apache Commons Compress for improved zip file handling in reader.ZipStreamStrategy --- build.gradle | 1 + .../ro_crate/reader/ZipStreamStrategy.java | 34 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index f7fcd613..2522dd13 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,7 @@ dependencies { implementation group: 'commons-io', name: 'commons-io', version: '2.19.0' // read from and write to zip files implementation group: 'net.lingala.zip4j', name: 'zip4j', version: '2.11.5' + implementation 'org.apache.commons:commons-compress:1.27.1' // compare json documents in tests implementation 'com.github.fslev:json-compare:7.0' // url validator diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index 03286a29..01f6504d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -4,15 +4,14 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; + +import java.io.*; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Enumeration; import java.util.UUID; -import net.lingala.zip4j.io.inputstream.ZipInputStream; -import net.lingala.zip4j.model.LocalFileHeader; + +import org.apache.commons.compress.archivers.zip.*; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; @@ -99,26 +98,27 @@ private void readCrate(InputStream stream) { FileUtils.forceMkdir(folder); } - LocalFileHeader localFileHeader; - int readLen; - byte[] readBuffer = new byte[4096]; - try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { - while ((localFileHeader = zipInputStream.getNextEntry()) != null) { - String fileName = localFileHeader.getFileName(); + ZipFile.Builder zipFileBuilder = new ZipFile.Builder() + .setByteArray(stream.readAllBytes()) + .setIgnoreLocalFileHeader(true) + .setOpenOptions(StandardOpenOption.READ); + try (ZipFile zipFile = zipFileBuilder.get()) { + Enumeration filesInZip = zipFile.getEntries(); + while (filesInZip.hasMoreElements()) { + ZipArchiveEntry entry = filesInZip.nextElement(); + String fileName = entry.getName(); File extractedFile = new File(folder, fileName).getCanonicalFile(); if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { throw new IOException("Entry is outside of target directory: " + fileName); } - if (localFileHeader.isDirectory()) { + if (entry.isDirectory()) { FileUtils.forceMkdir(extractedFile); continue; } FileUtils.forceMkdir(extractedFile.getParentFile()); try (OutputStream outputStream = new FileOutputStream(extractedFile)) { - while ((readLen = zipInputStream.read(readBuffer)) != -1) { - outputStream.write(readBuffer, 0, readLen); - } + zipFile.getInputStream(entry).transferTo(outputStream); } } } From a704a0b6a50852790a0fd8afd3248d144072467d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 12:09:44 +0200 Subject: [PATCH 37/79] Revert "fix: integrate Apache Commons Compress for improved zip file handling in reader.ZipStreamStrategy" This reverts commit 5dd5421d77fd165fa316084d1bfd7cc0e7698af4. --- build.gradle | 1 - .../ro_crate/reader/ZipStreamStrategy.java | 34 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index 2522dd13..f7fcd613 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,6 @@ dependencies { implementation group: 'commons-io', name: 'commons-io', version: '2.19.0' // read from and write to zip files implementation group: 'net.lingala.zip4j', name: 'zip4j', version: '2.11.5' - implementation 'org.apache.commons:commons-compress:1.27.1' // compare json documents in tests implementation 'com.github.fslev:json-compare:7.0' // url validator diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index 01f6504d..03286a29 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -4,14 +4,15 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; - -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Enumeration; import java.util.UUID; - -import org.apache.commons.compress.archivers.zip.*; +import net.lingala.zip4j.io.inputstream.ZipInputStream; +import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; @@ -98,27 +99,26 @@ private void readCrate(InputStream stream) { FileUtils.forceMkdir(folder); } + LocalFileHeader localFileHeader; + int readLen; + byte[] readBuffer = new byte[4096]; - ZipFile.Builder zipFileBuilder = new ZipFile.Builder() - .setByteArray(stream.readAllBytes()) - .setIgnoreLocalFileHeader(true) - .setOpenOptions(StandardOpenOption.READ); - try (ZipFile zipFile = zipFileBuilder.get()) { - Enumeration filesInZip = zipFile.getEntries(); - while (filesInZip.hasMoreElements()) { - ZipArchiveEntry entry = filesInZip.nextElement(); - String fileName = entry.getName(); + try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { + while ((localFileHeader = zipInputStream.getNextEntry()) != null) { + String fileName = localFileHeader.getFileName(); File extractedFile = new File(folder, fileName).getCanonicalFile(); if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { throw new IOException("Entry is outside of target directory: " + fileName); } - if (entry.isDirectory()) { + if (localFileHeader.isDirectory()) { FileUtils.forceMkdir(extractedFile); continue; } FileUtils.forceMkdir(extractedFile.getParentFile()); try (OutputStream outputStream = new FileOutputStream(extractedFile)) { - zipFile.getInputStream(entry).transferTo(outputStream); + while ((readLen = zipInputStream.read(readBuffer)) != -1) { + outputStream.write(readBuffer, 0, readLen); + } } } } From 586276b8544d3324877462ee3a43c930a5ddaedc Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 12:44:42 +0200 Subject: [PATCH 38/79] Revert "fix: invalid entities from our providers should be logged and not written" This reverts commit 86654f65fbb5dcaf1f6df75ace43524c93331ed4. --- .../externalproviders/dataentities/ImportFromZenodo.java | 4 ++-- .../externalproviders/personprovider/OrcidProvider.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java index 2cc6680a..4a99ee8a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java @@ -112,12 +112,12 @@ private static void addToCrateFromZotero(String url, Crate crate) { for (var entity : graph) { if (entity.get("@id").asText().equals(mainId)) { var dataEntity = new DataEntity.DataEntityBuilder() - .setAllIfValid((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addDataEntity(dataEntity); } else { // here we have to think of a way to differentiate between data and contextual entities. var contextualEntity = new ContextualEntity.ContextualEntityBuilder() - .setAllIfValid((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addContextualEntity(contextualEntity); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java index dd0b802b..17207993 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java @@ -72,7 +72,7 @@ public static PersonEntity getPerson(String url) { node.set(element.getKey(), element.getValue()); } } - return new PersonEntity.PersonEntityBuilder().setAllIfValid(node).build(); + return new PersonEntity.PersonEntityBuilder().setAllUnsafe(node).build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); logger.error(errorMessage); From eac3a7591326944ce766da5db14c36e803d0bcd3 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 13:09:39 +0200 Subject: [PATCH 39/79] fix: propagate IOException in crate reading and writing methods --- .../edu/kit/datamanager/ro_crate/reader/CrateReader.java | 3 ++- .../datamanager/ro_crate/reader/GenericReaderStrategy.java | 5 +++-- .../edu/kit/datamanager/ro_crate/writer/CrateWriter.java | 4 +++- .../datamanager/ro_crate/writer/GenericWriterStrategy.java | 4 +++- .../kit/datamanager/ro_crate/crate/BuilderSpec12Test.java | 3 ++- .../kit/datamanager/ro_crate/crate/ReadAndWriteTest.java | 2 +- .../datamanager/ro_crate/crate/TestRemoveAddContext.java | 3 ++- .../ro_crate/crate/preview/PreviewCrateTest.java | 6 +++--- .../kit/datamanager/ro_crate/reader/FolderReaderTest.java | 2 +- .../ro_crate/reader/RoCrateReaderSpec12Test.java | 3 ++- .../edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java | 2 +- 11 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 6586f1ed..7dcadbdc 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -83,7 +84,7 @@ public CrateReader(GenericReaderStrategy strategy) { * @param location the location of the ro-crate to be read * @return the read RO-crate */ - public RoCrate readCrate(T location) { + public RoCrate readCrate(T location) throws IOException { // get the ro-crate-metadata.json ObjectNode metadataJson = strategy.readMetadataJson(location); // get the content of the crate diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java index c3539b17..03f6b9c9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.File; +import java.io.IOException; /** * Generic interface for the strategy of the reader class. @@ -16,7 +17,7 @@ public interface GenericReaderStrategy { * @param location the location to read from * @return the parsed metadata.json as ObjectNode */ - ObjectNode readMetadataJson(T location); + ObjectNode readMetadataJson(T location) throws IOException; /** * Read the content from the given location. @@ -24,5 +25,5 @@ public interface GenericReaderStrategy { * @param location the location to read from * @return the content as a File */ - File readContent(T location); + File readContent(T location) throws IOException; } \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java index 440be0c4..4ee1be6b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -4,6 +4,8 @@ import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; +import java.io.IOException; + /** * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) @@ -22,7 +24,7 @@ public CrateWriter(GenericWriterStrategy strategy) { * @param crate the crate to write. * @param destination the location where the crate should be written. */ - public void save(Crate crate, DESTINATION destination) { + public void save(Crate crate, DESTINATION destination) throws IOException { Validator defaultValidation = new Validator(new JsonSchemaValidation()); defaultValidation.validate(crate); this.strategy.save(crate, destination); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java index 6306b576..c0e79779 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java @@ -2,6 +2,8 @@ import edu.kit.datamanager.ro_crate.Crate; +import java.io.IOException; + /** * Generic interface for the strategy of the writer class. * This allows for flexible output types when implementing different writing strategies. @@ -15,5 +17,5 @@ public interface GenericWriterStrategy { * @param crate The crate to save * @param destination The destination where the crate should be saved */ - void save(Crate crate, DESTINATION destination); + void save(Crate crate, DESTINATION destination) throws IOException; } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java index eddd93ee..270127af 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; @@ -37,7 +38,7 @@ void testAppendConformsTo() throws URISyntaxException { } @Test - void testModificationOfDraftCrate() throws URISyntaxException { + void testModificationOfDraftCrate() throws URISyntaxException, IOException { String path = this.getClass().getResource("/crates/spec-1.2-DRAFT/minimal-with-conformsTo-Array").getPath(); RoCrate crate = Readers.newFolderReader().readCrate(path); Collection existingProfiles = crate.getProfiles(); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java index 3c2e49b5..ca742ded 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java @@ -49,7 +49,7 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { @SuppressWarnings("DataFlowIssue") @Test - void testReadCrateWithHasPartHierarchy() { + void testReadCrateWithHasPartHierarchy() throws IOException { CrateReader reader = Readers.newFolderReader(); RoCrate crate = reader.readCrate(ReadAndWriteTest.class.getResource("/crates/hasPartHierarchy").getPath()); assertEquals(1, crate.getAllContextualEntities().size()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java index b1f6e21b..31c0dc8e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; import java.util.Objects; import java.util.Set; @@ -17,7 +18,7 @@ public class TestRemoveAddContext { private RoCrate crateWithComplexContext; @BeforeEach - void setup() { + void setup() throws IOException { String crateManifestPath = "/crates/extendedContextExample/"; crateManifestPath = Objects.requireNonNull(TestRemoveAddContext.class.getResource(crateManifestPath)).getPath(); this.crateWithComplexContext = Readers.newFolderReader().readCrate(crateManifestPath); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index 1ecc2062..92d70449 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -22,7 +22,7 @@ public class PreviewCrateTest { @Test - void testAutomaticPreview(@TempDir Path temp) { + void testAutomaticPreview(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate1"); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new AutomaticPreview()) @@ -33,7 +33,7 @@ void testAutomaticPreview(@TempDir Path temp) { } @Test - void testAutomaticPreviewAddingLater(@TempDir Path temp) { + void testAutomaticPreviewAddingLater(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate2"); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(null)//disable preview to allow to compare folders before and after @@ -47,7 +47,7 @@ void testAutomaticPreviewAddingLater(@TempDir Path temp) { } @Test - void testCustomPreview(@TempDir Path temp) { + void testCustomPreview(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate1"); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new CustomPreview()) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index a7143862..8375ba36 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -20,7 +20,7 @@ class FolderReaderTest implements CommonReaderTest { @Override - public void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newFolderWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isDirectory()); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java index 7b1df7df..68094231 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.util.stream.StreamSupport; import org.junit.jupiter.api.Test; @@ -27,7 +28,7 @@ public class RoCrateReaderSpec12Test { * https://www.researchobject.org/ro-crate/1.2-DRAFT/profiles.html#declaring-conformance-of-an-ro-crate-profile */ @Test - void testReadingCrateWithConformsToArray() { + void testReadingCrateWithConformsToArray() throws IOException { String path = this.getClass().getResource("/crates/spec-1.2-DRAFT/minimal-with-conformsTo-Array").getPath(); Crate crate = Readers.newFolderReader().readCrate(path); JsonNode conformsTo = crate.getJsonDescriptor().getProperty("conformsTo"); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index a4616f8e..7d71d94e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -13,7 +13,7 @@ class ZipReaderTest implements ElnFileFormatTest { @Override - public void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isFile()); } From 62b1b0c63dc55aede79fd433db85a277717832b6 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 13:10:30 +0200 Subject: [PATCH 40/79] refactor: setAll method now uses setAllIfValid to avoid code duplication --- .../kit/datamanager/ro_crate/entities/AbstractEntity.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 3ec7bb9f..34889d21 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -572,11 +572,7 @@ public T addIdFromCollectionOfEntities(String name, Collection e */ @Deprecated(since = "2.1.0", forRemoval = true) public T setAll(ObjectNode properties) { - if (AbstractEntity.entityValidation.entityValidation(properties)) { - this.properties = properties; - this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties)); - } - return self(); + return setAllIfValid(properties); } /** From d3f798692a13901fd44eaf2d356573e2ac2a87cb Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 13:30:36 +0200 Subject: [PATCH 41/79] tests: add blacklist functionality for unsupported ELN files in tests --- .../ro_crate/reader/ZipStreamStrategy.java | 80 ++++++++----------- .../ro_crate/reader/ElnFileFormatTest.java | 27 ++++++- .../ro_crate/reader/ZipStreamReaderTest.java | 14 ++++ 3 files changed, 72 insertions(+), 49 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index 03286a29..f6ed4577 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -85,53 +85,49 @@ public boolean isExtracted() { * * @param stream The input stream. */ - private void readCrate(InputStream stream) { - try { - File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.exists()) { - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - } else { - FileUtils.forceMkdir(folder); + private void readCrate(InputStream stream) throws IOException { + File folder = temporaryFolder.toFile(); + // ensure the directory is clean + if (folder.exists()) { + if (folder.isDirectory()) { + FileUtils.cleanDirectory(folder); + } else if (folder.isFile()) { + FileUtils.delete(folder); } + } else { + FileUtils.forceMkdir(folder); + } - LocalFileHeader localFileHeader; - int readLen; - byte[] readBuffer = new byte[4096]; + LocalFileHeader localFileHeader; + int readLen; + byte[] readBuffer = new byte[4096]; - try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { - while ((localFileHeader = zipInputStream.getNextEntry()) != null) { - String fileName = localFileHeader.getFileName(); - File extractedFile = new File(folder, fileName).getCanonicalFile(); - if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { - throw new IOException("Entry is outside of target directory: " + fileName); - } - if (localFileHeader.isDirectory()) { - FileUtils.forceMkdir(extractedFile); - continue; - } - FileUtils.forceMkdir(extractedFile.getParentFile()); - try (OutputStream outputStream = new FileOutputStream(extractedFile)) { - while ((readLen = zipInputStream.read(readBuffer)) != -1) { - outputStream.write(readBuffer, 0, readLen); - } + try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { + while ((localFileHeader = zipInputStream.getNextEntry()) != null) { + String fileName = localFileHeader.getFileName(); + File extractedFile = new File(folder, fileName).getCanonicalFile(); + if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { + throw new IOException("Entry is outside of target directory: " + fileName); + } + if (localFileHeader.isDirectory()) { + FileUtils.forceMkdir(extractedFile); + continue; + } + FileUtils.forceMkdir(extractedFile.getParentFile()); + try (OutputStream outputStream = new FileOutputStream(extractedFile)) { + while ((readLen = zipInputStream.read(readBuffer)) != -1) { + outputStream.write(readBuffer, 0, readLen); } } } - this.isExtracted = true; - // register deletion on exit - FileUtils.forceDeleteOnExit(folder); - } catch (IOException ex) { - logger.error("Failed to read crate from input stream.", ex); } + this.isExtracted = true; + // register deletion on exit + FileUtils.forceDeleteOnExit(folder); } @Override - public ObjectNode readMetadataJson(InputStream stream) { + public ObjectNode readMetadataJson(InputStream stream) throws IOException { if (!isExtracted) { this.readCrate(stream); } @@ -152,17 +148,11 @@ public ObjectNode readMetadataJson(InputStream stream) { .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); } - - try { - return objectMapper.readTree(jsonMetadata).deepCopy(); - } catch (IOException e) { - logger.error("Failed to deserialize crate metadata.", e); - return null; - } + return objectMapper.readTree(jsonMetadata).deepCopy(); } @Override - public File readContent(InputStream stream) { + public File readContent(InputStream stream) throws IOException { if (!isExtracted) { this.readCrate(stream); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java index 3e06fc7f..bc10848c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -19,6 +19,20 @@ public interface ElnFileFormatTest< > extends TestableReaderStrategy { + /** + * Some readers may not be able to read a subset of eln files, + * e.g. because a zip file may not be readable in streaming mode. + *

+ * An implementation test may use this methode to provide a subset of the + * test cases where an IOException is expected. + * + * @param input the input to test for presence in the blacklist + * @return true if the input is in the blacklist, false otherwise + */ + default boolean isInBlacklist(String input) { + return false; + } + /** * ELN Crates are zip files not fully compatible with the Ro-Crate standard * in the sense that they must contain a single subfolder in the zip file @@ -46,9 +60,14 @@ default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOExcept FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); assertTrue(elnFile.toFile().exists()); - // Read the crate from the downloaded file - Crate read = this.readCrate(elnFile); - assertNotNull(read); - assertFalse(read.getAllDataEntities().isEmpty()); + if (!isInBlacklist(urlStr)) { + // Read the crate from the downloaded file + Crate read = this.readCrate(elnFile); + assertNotNull(read); + assertFalse(read.getAllDataEntities().isEmpty()); + } else { + // If the file is in the blacklist, we expect an IOException + assertThrows(IOException.class, () -> this.readCrate(elnFile)); + } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index 17b37247..a6cca8e6 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -5,6 +5,7 @@ import java.io.*; import java.nio.file.Path; +import java.util.Set; import static org.junit.jupiter.api.Assertions.*; @@ -13,6 +14,19 @@ class ZipStreamReaderTest implements CommonReaderTest, ElnFileFormatTest { + /** + * At the point of writing this test, + * these files are in a zip format which cannot be read in streaming mode + */ + @Override + public boolean isInBlacklist(String input) { + return Set.of( + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" + ) + .contains(input); + } + @Override public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); From cca2cd5672eb57c8825126d42a325dbd0b25602d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:18:06 +0200 Subject: [PATCH 42/79] tests: remove possible side effects between tests in LearnByExampleTest.java --- .../ro_crate/examples/LearnByExampleTest.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java index aab61632..64733e89 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java @@ -42,20 +42,22 @@ public class LearnByExampleTest { /** * This creates a valid, empty RO-Crate builder. */ - static final RoCrate.RoCrateBuilder STARTER_CRATE = new RoCrate.RoCrateBuilder( - "name", - "description", - "2025", - "licenseIdentifier" - ); + static RoCrate.RoCrateBuilder NEW_STARTER_CRATE() { + return new RoCrate.RoCrateBuilder( + "name", + "description", + "2025", + "licenseIdentifier" + ); + } /** * Calling the `build()` method on the builder creates a valid RO-Crate. - * Run this test to view the STARTER_CRATE JSON in the console. + * Run this test to view the NEW_STARTER_CRATE() JSON in the console. */ @Test void aSimpleCrate() { - RoCrate almostEmptyCrate = STARTER_CRATE.build(); + RoCrate almostEmptyCrate = NEW_STARTER_CRATE().build(); assertNotNull(almostEmptyCrate); HelpFunctions.prettyPrintJsonString(almostEmptyCrate.getJsonMetadata()); } @@ -72,7 +74,7 @@ void aSimpleCrate() { */ @Test void addingYourFirstEntity() { - RoCrate myFirstCrate = STARTER_CRATE + RoCrate myFirstCrate = NEW_STARTER_CRATE() // We can add new terms to our crate. The terms we can use are called "context". .addValuePairToContext("Station", "www.station.com") // We can also add whole contexts to our crate. @@ -115,7 +117,7 @@ void addingYourFirstEntity() { */ @Test void specializingYourFirstEntity() { - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addDataEntity( // Let's do something custom: new DataEntity.DataEntityBuilder() @@ -148,7 +150,7 @@ void referencingFilesOnTheWeb() { // Let's say this is the file we would like to point at with an entity. String lovelyFile = "https://github.com/kit-data-manager/ro-crate-java/issues/5"; - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addDataEntity( // Build our entity to point to the file: new FileEntity.FileEntityBuilder() @@ -187,7 +189,7 @@ void includingFilesIntoTheCrateFolder(@TempDir Path tempDir) throws IOException // But in the crate we want it to be String seriousExperimentFile = "fantastic-experiment/2025-01-01.csv"; - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addDataEntity( // Build our entity to point to the file: new FileEntity.FileEntityBuilder() @@ -236,7 +238,7 @@ void addingContextualEntities() { PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addContextualEntity(person) .addContextualEntity(organization) .build(); @@ -263,7 +265,7 @@ void writingAndReadingCrates(@TempDir Path tempDir) throws IOException { PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addContextualEntity(person) .addContextualEntity(organization) .build(); @@ -324,7 +326,7 @@ void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addContextualEntity(person) .addContextualEntity(organization) .build(); @@ -368,7 +370,7 @@ void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { */ @Test void humanReadableContent() { - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .setPreview(new AutomaticPreview()) .build(); @@ -380,11 +382,13 @@ void humanReadableContent() { * Therefore, the constructor is a bit more complicated. */ @Test - void staticPreview(@TempDir Path tempDir) { + void staticPreview(@TempDir Path tempDir) throws IOException { File mainPreviewHtml = tempDir.resolve("mainPreview.html").toFile(); File additionalFilesDirectory = tempDir.resolve("additionalFiles").toFile(); + FileUtils.forceMkdir(additionalFilesDirectory); + FileUtils.touch(mainPreviewHtml); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .setPreview(new StaticPreview(mainPreviewHtml, additionalFilesDirectory)) .build(); @@ -408,7 +412,7 @@ void validation() { String schemaPath = schemaUrl.getPath(); // This crate for sure is not a workflow, so validation will fail. - RoCrate crate = STARTER_CRATE.build(); + RoCrate crate = NEW_STARTER_CRATE().build(); // And now do the validation. Validator validator = new Validator(new JsonSchemaValidation(schemaPath)); From 6fd9ac26fb19d8cb457fde4eda4ae5d4b176532a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:35:05 +0200 Subject: [PATCH 43/79] refactor: pull DataEntity saveToStream logic into writer.ZipStreamStrategy The zip dependency is too widely spread over different classes. We use it for reading and writing, so it should stay in classes which handle reading and writing. --- .../ro_crate/entities/data/DataEntity.java | 17 ---------- .../ro_crate/entities/data/DataSetEntity.java | 13 -------- .../ro_crate/writer/ZipStreamStrategy.java | 33 ++++++++++++++++++- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index e6e28f8f..28d043c9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -5,7 +5,6 @@ import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl; -import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; import java.io.IOException; @@ -15,7 +14,6 @@ import java.util.List; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -72,21 +70,6 @@ public void saveToZip(ZipFile zipFile) throws ZipException { } } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a zip archive. - * - * @param zipStream The zip output stream where it should be written. - * @throws ZipException when something goes wrong with the writing to the - * zip file. - * @throws IOException If opening the file input stream fails. - */ - public void saveToStream(ZipOutputStream zipStream) throws ZipException, IOException { - if (this.path != null) { - ZipUtil.addFileToZipStream(zipStream, this.path.toFile(), this.getId()); - } - } - /** * If the data entity contains a physical file. This method will write it * when the crate is being written to a folder. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index 832d9819..7ea4c789 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -4,14 +4,11 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import edu.kit.datamanager.ro_crate.entities.serializers.HasPartSerializer; -import edu.kit.datamanager.ro_crate.util.ZipUtil; -import java.io.IOException; import java.util.HashSet; import java.util.Set; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; /** @@ -53,16 +50,6 @@ public void saveToZip(ZipFile zipFile) throws ZipException { } } - @Override - public void saveToStream(ZipOutputStream zipOutputStream) throws IOException { - if (this.getPath() != null) { - ZipUtil.addFolderToZipStream( - zipOutputStream, - this.getPath().toAbsolutePath().toString(), - this.getId()); - } - } - public void addToHasPart(String id) { this.hasPart.add(id); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index 818c064c..2a9d1fbb 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -12,6 +12,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; + +import edu.kit.datamanager.ro_crate.util.ZipUtil; +import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.slf4j.Logger; @@ -39,7 +42,7 @@ public void save(Crate crate, OutputStream destination) { private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { for (DataEntity dataEntity : crate.getAllDataEntities()) { try { - dataEntity.saveToStream(zipStream); + saveToStream(dataEntity, zipStream); } catch (IOException e) { logger.error("Could not save {} to zip stream!", dataEntity.getId(), e); } @@ -74,4 +77,32 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { logger.error("Exception writing ro-crate-metadata.json file to zip.", e); } } + + /** + * If the data entity contains a physical file. This method will write it + * when the crate is being written to a zip archive. + * + * @param zipStream The zip output stream where it should be written. + * @throws ZipException when something goes wrong with the writing to the + * zip file. + * @throws IOException If opening the file input stream fails. + */ + private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws ZipException, IOException { + if (entity == null) { + return; + } + + boolean isDirectory = entity.getPath().toFile().isDirectory(); + if (isDirectory) { + ZipUtil.addFolderToZipStream( + zipStream, + entity.getPath().toAbsolutePath().toString(), + entity.getId()); + } else { + ZipUtil.addFileToZipStream( + zipStream, + entity.getPath().toFile(), + entity.getId()); + } + } } From b58d6bdd9d054d277808e9890f9dfcc6562dc780 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:38:38 +0200 Subject: [PATCH 44/79] refactor: propagate IOExceptions properly in writer.ZipStreamStrategy.java --- .../ro_crate/writer/ZipStreamStrategy.java | 57 ++++++++----------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index 2a9d1fbb..db69cb3a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -29,52 +29,41 @@ public class ZipStreamStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); @Override - public void save(Crate crate, OutputStream destination) { + public void save(Crate crate, OutputStream destination) throws IOException { try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { saveMetadataJson(crate, zipFile); saveDataEntities(crate, zipFile); - } catch (IOException e) { - // can not close ZipOutputStream (threw Exception) - logger.error("Failed to save ro-crate to zip stream.", e); } } - private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { + private void saveDataEntities(Crate crate, ZipOutputStream zipStream) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - saveToStream(dataEntity, zipStream); - } catch (IOException e) { - logger.error("Could not save {} to zip stream!", dataEntity.getId(), e); - } + saveToStream(dataEntity, zipStream); } } - private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { - try { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - // write the ro-crate-metadata + private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOException { + // write the metadata.json file + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + // write the ro-crate-metadata - byte[] buff = new byte[4096]; - int readLen; - zipStream.putNextEntry(zipParameters); - try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { - while ((readLen = inputStream.read(buff)) != -1) { - zipStream.write(buff, 0, readLen); - } + byte[] buff = new byte[4096]; + int readLen; + zipStream.putNextEntry(zipParameters); + try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { + while ((readLen = inputStream.read(buff)) != -1) { + zipStream.write(buff, 0, readLen); } - zipStream.closeEntry(); + } + zipStream.closeEntry(); - if (crate.getPreview() != null) { - crate.getPreview().saveAllToStream(str, zipStream); - } - } catch (IOException e) { - logger.error("Exception writing ro-crate-metadata.json file to zip.", e); + if (crate.getPreview() != null) { + crate.getPreview().saveAllToStream(str, zipStream); } } @@ -87,7 +76,7 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { * zip file. * @throws IOException If opening the file input stream fails. */ - private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws ZipException, IOException { + private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws IOException { if (entity == null) { return; } From 6e1dae62955c437b824981319066dd8fd44537cf Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:40:41 +0200 Subject: [PATCH 45/79] refactor: propagate IOExceptions properly in writer.FolderStrategy --- .../ro_crate/writer/FolderStrategy.java | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index b2585637..6a3c85c0 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -26,38 +26,30 @@ public class FolderStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); @Override - public void save(Crate crate, String destination) { + public void save(Crate crate, String destination) throws IOException { File file = new File(destination); - try { - FileUtils.forceMkdir(file); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + FileUtils.forceMkdir(file); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); - File json = new File(destination, "ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(inputStream, json); - inputStream.close(); - // save also the preview files to the crate destination - if (crate.getPreview() != null) { - crate.getPreview().saveAllToFolder(file); - } - for (var e : crate.getUntrackedFiles()) { - if (e.isDirectory()) { - FileUtils.copyDirectoryToDirectory(e, file); - } else { - FileUtils.copyFileToDirectory(e, file); - } + File json = new File(destination, "ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(inputStream, json); + inputStream.close(); + // save also the preview files to the crate destination + if (crate.getPreview() != null) { + crate.getPreview().saveAllToFolder(file); + } + for (var e : crate.getUntrackedFiles()) { + if (e.isDirectory()) { + FileUtils.copyDirectoryToDirectory(e, file); + } else { + FileUtils.copyFileToDirectory(e, file); } - } catch (IOException e) { - logger.error("Error creating destination directory!", e); } for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.savetoFile(file); - } catch (IOException e) { - logger.error("Cannot save " + dataEntity.getId() + " to destination folder!", e); - } + dataEntity.savetoFile(file); } } } From 82cee04865cd23f9cddd2395e396b9286eb39820 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:42:21 +0200 Subject: [PATCH 46/79] refactor: propagate IOExceptions properly in writer.ZipStrategy --- .../ro_crate/writer/ZipStrategy.java | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index 6a11df87..00b920d2 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -25,44 +25,33 @@ public class ZipStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); @Override - public void save(Crate crate, String destination) { + public void save(Crate crate, String destination) throws IOException { try (ZipFile zipFile = new ZipFile(destination)) { saveMetadataJson(crate, zipFile); saveDataEntities(crate, zipFile); - } catch (IOException e) { - // can not close ZipFile (threw Exception) - logger.error("Failed to write ro-crate to destination " + destination + ".", e); } } - private void saveDataEntities(Crate crate, ZipFile zipFile) { + private void saveDataEntities(Crate crate, ZipFile zipFile) throws ZipException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.saveToZip(zipFile); - } catch (ZipException e) { - logger.error("Could not save " + dataEntity.getId() + " to zip file!", e); - } + dataEntity.saveToZip(zipFile); } } - private void saveMetadataJson(Crate crate, ZipFile zipFile) { + private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { // write the metadata.json file ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); - try { - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - } - if (crate.getPreview() != null) { - crate.getPreview().saveAllToZip(zipFile); - } - } catch (IOException e) { - logger.error("Exception writing ro-crate-metadata.json file to zip.", e); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { + // write the ro-crate-metadata + zipFile.addStream(inputStream, zipParameters); + } + if (crate.getPreview() != null) { + crate.getPreview().saveAllToZip(zipFile); } } } From bee2917fedd32822f2720632dc55177c04c4e22b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:26:30 +0200 Subject: [PATCH 47/79] refactor: pull DataEntity saveToZip logic into writer.ZipStrategy --- .../ro_crate/entities/data/DataEntity.java | 19 --------------- .../ro_crate/entities/data/DataSetEntity.java | 13 ----------- .../ro_crate/writer/ZipStrategy.java | 23 ++++++++++++++++--- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 28d043c9..be4253b1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -12,9 +12,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; /** @@ -54,22 +51,6 @@ public void addAuthorId(String id) { this.addIdProperty("author", id); } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a zip archive. - * - * @param zipFile the zipFile where it should be written. - * @throws ZipException when something goes wrong with the writing to the - * zip file. - */ - public void saveToZip(ZipFile zipFile) throws ZipException { - if (this.path != null) { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(this.getId()); - zipFile.addFile(this.path.toFile(), zipParameters); - } - } - /** * If the data entity contains a physical file. This method will write it * when the crate is being written to a folder. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index 7ea4c789..2ef078ff 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -7,9 +7,6 @@ import java.util.HashSet; import java.util.Set; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; /** * A helping class for the creating of Data entities of type Dataset. @@ -40,16 +37,6 @@ public void removeFromHasPart(String str) { this.hasPart.remove(str); } - @Override - public void saveToZip(ZipFile zipFile) throws ZipException { - if (this.getPath() != null) { - ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(this.getId()); - parameters.setIncludeRootFolder(false); - zipFile.addFolder(this.getPath().toFile(), parameters); - } - } - public void addToHasPart(String id) { this.hasPart.add(id); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index 00b920d2..0c8634b1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -6,7 +6,6 @@ import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.model.ZipParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,9 +31,9 @@ public void save(Crate crate, String destination) throws IOException { } } - private void saveDataEntities(Crate crate, ZipFile zipFile) throws ZipException { + private void saveDataEntities(Crate crate, ZipFile zipFile) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - dataEntity.saveToZip(zipFile); + this.saveToZip(dataEntity, zipFile); } } @@ -54,4 +53,22 @@ private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { crate.getPreview().saveAllToZip(zipFile); } } + + private void saveToZip(DataEntity entity, ZipFile zipFile) throws IOException { + if (entity == null || entity.getPath() == null) { + return; + } + + boolean isDirectory = entity.getPath().toFile().isDirectory(); + if (isDirectory) { + ZipParameters parameters = new ZipParameters(); + parameters.setRootFolderNameInZip(entity.getId()); + parameters.setIncludeRootFolder(false); + zipFile.addFolder(entity.getPath().toFile(), parameters); + } else { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(entity.getId()); + zipFile.addFile(entity.getPath().toFile(), zipParameters); + } + } } From 08a5f10a47c23e3b6cb3e3b50745cef358ced51c Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:27:07 +0200 Subject: [PATCH 48/79] refactor: remove outdated saveToStream method documentation in writer.ZipStreamStrategy --- .../datamanager/ro_crate/writer/ZipStreamStrategy.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index db69cb3a..b97b3c2a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -67,15 +67,6 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOE } } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a zip archive. - * - * @param zipStream The zip output stream where it should be written. - * @throws ZipException when something goes wrong with the writing to the - * zip file. - * @throws IOException If opening the file input stream fails. - */ private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws IOException { if (entity == null) { return; From 0fc94e78a3718c39e18b8f4986580e2abdb9177b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:33:32 +0200 Subject: [PATCH 49/79] refactor: move savetoFile logic from DataEntity to FolderStrategy --- .../ro_crate/entities/data/DataEntity.java | 20 ------------------- .../ro_crate/writer/FolderStrategy.java | 12 ++++++++++- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index be4253b1..864d3044 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -6,13 +6,10 @@ import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl; -import java.io.File; -import java.io.IOException; import java.net.URI; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import org.apache.commons.io.FileUtils; /** * The base class of every data entity. @@ -51,23 +48,6 @@ public void addAuthorId(String id) { this.addIdProperty("author", id); } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a folder. - * - * @param file the folder location where the entity should be written. - * @throws IOException if something goes wrong with the writing. - */ - public void savetoFile(File file) throws IOException { - if (this.getPath() != null) { - if (this.getPath().toFile().isDirectory()) { - FileUtils.copyDirectory(this.getPath().toFile(), file.toPath().resolve(this.getId()).toFile()); - } else { - FileUtils.copyFile(this.getPath().toFile(), file.toPath().resolve(this.getId()).toFile()); - } - } - } - @JsonIgnore public Path getPath() { return path; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index 6a3c85c0..8fa5a6bb 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -49,7 +49,17 @@ public void save(Crate crate, String destination) throws IOException { } } for (DataEntity dataEntity : crate.getAllDataEntities()) { - dataEntity.savetoFile(file); + savetoFile(dataEntity, file); + } + } + + private void savetoFile(DataEntity entity, File file) throws IOException { + if (entity.getPath() != null) { + if (entity.getPath().toFile().isDirectory()) { + FileUtils.copyDirectory(entity.getPath().toFile(), file.toPath().resolve(entity.getId()).toFile()); + } else { + FileUtils.copyFile(entity.getPath().toFile(), file.toPath().resolve(entity.getId()).toFile()); + } } } } From e41cfcfba32f62f8025706cb2d246a607ae2b4e6 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:37:53 +0200 Subject: [PATCH 50/79] refactor: propagate IOException handling in reader.FolderStrategy and reader.ZipStrategy --- .../ro_crate/reader/FolderStrategy.java | 8 +--- .../ro_crate/reader/ZipStrategy.java | 45 ++++++++----------- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java index f6e235ed..4b07e01d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java @@ -17,15 +17,11 @@ public class FolderStrategy implements GenericReaderStrategy { @Override - public ObjectNode readMetadataJson(String location) { + public ObjectNode readMetadataJson(String location) throws IOException { Path metadata = new File(location).toPath().resolve("ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); ObjectNode objectNode = objectMapper.createObjectNode(); - try { - objectNode = objectMapper.readTree(metadata.toFile()).deepCopy(); - } catch (IOException e) { - e.printStackTrace(); - } + objectNode = objectMapper.readTree(metadata.toFile()).deepCopy(); return objectNode; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java index 74622b2f..d25a235e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -80,29 +80,25 @@ public boolean isExtracted() { return isExtracted; } - private void readCrate(String location) { - try { - File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - // extract - try (ZipFile zf = new ZipFile(location)) { - zf.extractAll(temporaryFolder.toAbsolutePath().toString()); - this.isExtracted = true; - } - // register deletion on exit - FileUtils.forceDeleteOnExit(folder); - } catch (IOException e) { - e.printStackTrace(); + private void readCrate(String location) throws IOException { + File folder = temporaryFolder.toFile(); + // ensure the directory is clean + if (folder.isDirectory()) { + FileUtils.cleanDirectory(folder); + } else if (folder.isFile()) { + FileUtils.delete(folder); } + // extract + try (ZipFile zf = new ZipFile(location)) { + zf.extractAll(temporaryFolder.toAbsolutePath().toString()); + this.isExtracted = true; + } + // register deletion on exit + FileUtils.forceDeleteOnExit(folder); } @Override - public ObjectNode readMetadataJson(String location) { + public ObjectNode readMetadataJson(String location) throws IOException { if (!isExtracted) { this.readCrate(location); } @@ -123,17 +119,12 @@ public ObjectNode readMetadataJson(String location) { .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); } - - try { - return objectMapper.readTree(jsonMetadata).deepCopy(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } + + return objectMapper.readTree(jsonMetadata).deepCopy(); } @Override - public File readContent(String location) { + public File readContent(String location) throws IOException { if (!isExtracted) { this.readCrate(location); } From bccbf0bb5fd0a266b6e99749297ad8a7333f3e96 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 17:49:27 +0200 Subject: [PATCH 51/79] tests: implement ELN style crate writer test for ZipWriterTest --- .../datamanager/ro_crate/HelpFunctions.java | 23 +++ ...eWriterTest.java => CommonWriterTest.java} | 159 ++---------------- .../ro_crate/writer/ElnFileFormatTest.java | 70 ++++++++ .../ro_crate/writer/FolderWriterTest.java | 6 +- .../writer/TestableWriterStrategy.java | 121 +++++++++++++ .../writer/ZipStreamStrategyTest.java | 4 +- .../ro_crate/writer/ZipWriterTest.java | 13 +- 7 files changed, 246 insertions(+), 150 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{CrateWriterTest.java => CommonWriterTest.java} (50%) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java diff --git a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java index 2c733d3e..24a57a07 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java @@ -17,6 +17,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -152,4 +154,25 @@ public static boolean compareTwoDir(File dir1, File dir2) throws IOException { } return true; } + + /** + * Prints the file tree of the given directory for debugging and understanding + * a test more quickly. + * + * @param directoryToPrint the directory to print + * @throws IOException if an error occurs while printing the file tree + */ + @SuppressWarnings("resource") + public static void printFileTree(Path directoryToPrint) throws IOException { + // Print all files recursively in a tree structure for debugging + System.out.printf("Files in %s:%n", directoryToPrint.getFileName().toString()); + Files.walk(directoryToPrint) + .forEach(path -> { + if (!path.toAbsolutePath().equals(directoryToPrint.toAbsolutePath())) { + int depth = path.relativize(directoryToPrint).getNameCount(); + String prefix = " ".repeat(depth); + System.out.printf("%s%s%s%n", prefix, "└── ", path.getFileName()); + } + }); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java similarity index 50% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java index 429fc450..0eae96c2 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java @@ -1,20 +1,14 @@ package edu.kit.datamanager.ro_crate.writer; -import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; -import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; -import net.lingala.zip4j.ZipFile; + import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.IOException; -import java.io.InputStream; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -22,16 +16,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -abstract class CrateWriterTest { - - /** - * Saves the crate with the writer fitting to this test class. - * - * @param crate the crate to save - * @param target the target path to the save location - * @throws IOException if an error occurs while saving the crate - */ - abstract protected void saveCrate(Crate crate, Path target) throws IOException; +interface CommonWriterTest extends TestableWriterStrategy { /** * Test where the writer needs to rename files or folders in order to make a valid crate. @@ -41,11 +26,11 @@ abstract class CrateWriterTest { * @throws IOException if an error occurs while writing the crate */ @Test - void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { + default void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Path correctCrate = tempDir.resolve("compare_with_me"); Path pathToFile = correctCrate.resolve("you-will-need-to-rename-this-file.ai"); Path pathToDir = correctCrate.resolve("you-will-need-to-rename-this-dir"); - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + createManualCrateStructure(correctCrate, pathToFile, pathToDir); Path writtenCrate = tempDir.resolve("written-crate"); Path extractionPath = tempDir.resolve("checkMe"); @@ -60,11 +45,11 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { ) .build(); this.saveCrate(builtCrate, writtenCrate); - this.ensureCrateIsExtractedIn(writtenCrate, extractionPath); + ensureCrateIsExtractedIn(writtenCrate, extractionPath); } - printFileTree(correctCrate); - printFileTree(extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); // The actual file name should **not** appear in the crate String fileName = pathToFile.getFileName().toString(); @@ -111,7 +96,7 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { * @throws IOException if an error occurs while writing the crate */ @Test - void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { + default void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { // We need a correct directory to compare with. // It is built manually to ensure we meet our expectations. // Reader-writer-consistency is tested at {@link CrateReaderTest} @@ -119,7 +104,7 @@ void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { Path pathToFile = correctCrate.resolve("cp7glop.ai"); Path pathToDir = correctCrate.resolve("lots_of_little_files"); - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + createManualCrateStructure(correctCrate, pathToFile, pathToDir); // Now use the builder to build the same crate independently. // The files will be reused (we need a place to take a copy from) @@ -130,9 +115,9 @@ void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { // extract the zip file to a temporary directory Path extractionPath = tempDir.resolve("extracted_for_testing"); - this.ensureCrateIsExtractedIn(pathToZip, extractionPath); - printFileTree(correctCrate); - printFileTree(extractionPath); + ensureCrateIsExtractedIn(pathToZip, extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); // compare the extracted directory with the correct one assertTrue(HelpFunctions.compareTwoDir( @@ -151,12 +136,12 @@ void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { * @throws IOException if an error occurs while writing the crate */ @Test - void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOException { + default void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOException { Path correctCrate = tempDir.resolve("compare_with_me"); Path pathToFile = correctCrate.resolve("cp7glop.ai"); Path pathToDir = correctCrate.resolve("lots_of_little_files"); - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + createManualCrateStructure(correctCrate, pathToFile, pathToDir); { // This file is not part of the crate, and should therefore not be present Path falseFile = correctCrate.resolve("new"); @@ -176,8 +161,8 @@ void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOExceptio // extract and compare Path extractionPath = tempDir.resolve("extracted_for_testing"); ensureCrateIsExtractedIn(pathToZip, extractionPath); - printFileTree(correctCrate); - printFileTree(extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); assertFalse(HelpFunctions.compareTwoDir( correctCrate.toFile(), @@ -187,116 +172,4 @@ void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOExceptio roCrate, "/json/crate/fileAndDir.json"); } - - /** - * Prints the file tree of the given directory for debugging and understanding - * a test more quickly. - * - * @param directoryToPrint the directory to print - * @throws IOException if an error occurs while printing the file tree - */ - @SuppressWarnings("resource") - protected static void printFileTree(Path directoryToPrint) throws IOException { - // Print all files recursively in a tree structure for debugging - System.out.printf("Files in %s:%n", directoryToPrint.getFileName().toString()); - Files.walk(directoryToPrint) - .forEach(path -> { - if (!path.toAbsolutePath().equals(directoryToPrint.toAbsolutePath())) { - int depth = path.relativize(directoryToPrint).getNameCount(); - String prefix = " ".repeat(depth); - System.out.printf("%s%s%s%n", prefix, "└── ", path.getFileName()); - } - }); - } - - /** - * Ensures the crate is in extracted form in the given path. - * - * @param pathToCrate the path to the crate, may not be a folder yet - * @param expectedPath the path where the crate should be in extracted form - * @throws IOException if an error occurs while extracting the crate - */ - protected void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { - try (ZipFile zf = new ZipFile(pathToCrate.toFile())) { - zf.extractAll(expectedPath.toFile().getAbsolutePath()); - } - } - - /** - * Creates a crate structure manually. - * - * @param correctCrate the path to the crate - * @param pathToFile the path to the file - * @param pathToDir the path to the directory - * @throws IOException if an error occurs while creating the crate structure - */ - protected void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { - FileUtils.forceMkdir(correctCrate.toFile()); - InputStream fileJson = ZipStreamStrategyTest.class - .getResourceAsStream("/json/crate/fileAndDir.json"); - Assertions.assertNotNull(fileJson); - // fill the directory with expected files and dirs - // starting with the .json of our crate - Path json = correctCrate.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - // create preview - PreviewGenerator.generatePreview(correctCrate.toFile().getAbsolutePath()); - // create the files and directories - FileUtils.writeStringToFile(pathToFile.toFile(), "content of Local File", Charset.defaultCharset()); - // creates the directory and a subdirectory - Path subdir = pathToDir.resolve("subdir"); - FileUtils.forceMkdir(subdir.toFile()); - FileUtils.writeStringToFile( - subdir.resolve("subsubfirst.txt").toFile(), - "content of subsub file in subsubdir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - pathToDir.resolve("first.txt").toFile(), - "content of first file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - pathToDir.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - pathToDir.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - } - - /** - * Creates a crate resembling the one we manually create in these tests. - * - * @param pathToFile the file to add - * @param pathToSubdir the directory to add - * @return the crate builder - */ - protected RoCrate.RoCrateBuilder getCrateWithFileAndDir(Path pathToFile, Path pathToSubdir) { - return new RoCrate.RoCrateBuilder( - "Example RO-Crate", - "The RO-Crate Root Data Entity", - "2024", - "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" - ) - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(pathToFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(pathToSubdir) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()); - } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java new file mode 100644 index 00000000..feb8dc06 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java @@ -0,0 +1,70 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +public interface ElnFileFormatTest extends TestableWriterStrategy { + + /** + * Write in ELN format style, meaning with a subfolder in the zip file. + * + * @param crate the crate to write + * @param target the target path to the save location + * @throws IOException if an error occurs + */ + void saveCrateElnStyle(Crate crate, Path target) throws IOException; + + @Test + default void testMakesElnStyleCrate(@TempDir Path tempDir) throws IOException { + // We need a correct directory to compare with. + // It is built manually to ensure we meet our expectations. + // Reader-writer-consistency is tested at {@link CrateReaderTest} + + // We compare the ELN style like this: + // tempDir + // └── compare_with_me + // └── crate-subfolder + // ├── ... + // └── extracted_for_testing + // └── crate-subfolder + // ├── ... + String crateName = "crate-subfolder"; + Path correctCrate = tempDir + .resolve("compare_with_me") + .resolve(crateName); + Path pathToFile = correctCrate.resolve("cp7glop.ai"); + Path pathToDir = correctCrate.resolve("lots_of_little_files"); + + createManualCrateStructure(correctCrate, pathToFile, pathToDir); + + // Now use the builder to build the same crate independently. + // The files will be reused (we need a place to take a copy from) + RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + + Path pathToZip = tempDir.resolve("%s.eln".formatted(crateName)); + this.saveCrateElnStyle(builtCrate, pathToZip); + + // extract the zip file to a temporary directory + Path extractionPath = tempDir.resolve("extracted_for_testing"); + ensureCrateIsExtractedIn(pathToZip, extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); + + // compare the extracted directory with the correct one + assertTrue(HelpFunctions.compareTwoDir( + correctCrate.toFile(), + extractionPath.toFile())); + HelpFunctions.compareCrateJsonToFileInResources( + builtCrate, + "/json/crate/fileAndDir.json"); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java index 7a469465..0c187029 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java @@ -11,16 +11,16 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderWriterTest extends CrateWriterTest { +class FolderWriterTest implements CommonWriterTest { @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newFolderWriter() .save(crate, target.toAbsolutePath().toString()); } @Override - protected void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { + public void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { FileUtils.copyDirectory(pathToCrate.toFile(), expectedPath.toFile()); } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java new file mode 100644 index 00000000..6cc41a30 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java @@ -0,0 +1,121 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; +import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; +import net.lingala.zip4j.ZipFile; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Assertions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.file.Path; + +/** + * Base Interface for methods required to test all writer strategies. + */ +interface TestableWriterStrategy { + /** + * Saves the crate with the writer fitting to this test class. + * + * @param crate the crate to save + * @param target the target path to the save location + * @throws IOException if an error occurs while saving the crate + */ + void saveCrate(Crate crate, Path target) throws IOException; + + /** + * Ensures the crate is in extracted form in the given path. + * + * @param pathToCrate the path to the crate, may not be a folder yet + * @param expectedPath the path where the crate should be in extracted form + * @throws IOException if an error occurs while extracting the crate + */ + default void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { + try (ZipFile zf = new ZipFile(pathToCrate.toFile())) { + zf.extractAll(expectedPath.toFile().getAbsolutePath()); + } + } + + /** + * Creates a crate structure manually. + * + * @param correctCrate the path to the crate + * @param pathToFile the path to the file + * @param pathToDir the path to the directory + * @throws IOException if an error occurs while creating the crate structure + */ + default void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { + FileUtils.forceMkdir(correctCrate.toFile()); + InputStream fileJson = ZipStreamStrategyTest.class + .getResourceAsStream("/json/crate/fileAndDir.json"); + Assertions.assertNotNull(fileJson); + // fill the directory with expected files and dirs + // starting with the .json of our crate + Path json = correctCrate.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + // create preview + PreviewGenerator.generatePreview(correctCrate.toFile().getAbsolutePath()); + // create the files and directories + FileUtils.writeStringToFile(pathToFile.toFile(), "content of Local File", Charset.defaultCharset()); + // creates the directory and a subdirectory + Path subdir = pathToDir.resolve("subdir"); + FileUtils.forceMkdir(subdir.toFile()); + FileUtils.writeStringToFile( + subdir.resolve("subsubfirst.txt").toFile(), + "content of subsub file in subsubdir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("first.txt").toFile(), + "content of first file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + } + + /** + * Creates a crate resembling the one we manually create in these tests. + * + * @param pathToFile the file to add + * @param pathToSubdir the directory to add + * @return the crate builder + */ + default RoCrate.RoCrateBuilder getCrateWithFileAndDir(Path pathToFile, Path pathToSubdir) { + return new RoCrate.RoCrateBuilder( + "Example RO-Crate", + "The RO-Crate Root Data Entity", + "2024", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(pathToFile) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(pathToSubdir) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index 283b306a..aa3ee5c6 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -8,10 +8,10 @@ /** * @author jejkal */ -class ZipStreamStrategyTest extends CrateWriterTest { +class ZipStreamStrategyTest implements CommonWriterTest { @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { try (FileOutputStream stream = new FileOutputStream(target.toFile())) { Writers.newZipStreamWriter().save(crate, stream); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 3b8b1fc0..80aa91ca 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -5,10 +5,19 @@ import edu.kit.datamanager.ro_crate.Crate; -class ZipWriterTest extends CrateWriterTest { +class ZipWriterTest implements + CommonWriterTest, + ElnFileFormatTest +{ @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipPathWriter() .save(crate, target.toAbsolutePath().toString()); } + + @Override + public void saveCrateElnStyle(Crate crate, Path target) throws IOException { + new CrateWriter<>(new ZipStrategy().usingElnStyle()) + .save(crate, target.toAbsolutePath().toString()); + } } From b46be7f9ce561bbec549ed3b449d9e91b53a9066 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 18:41:00 +0200 Subject: [PATCH 52/79] feat: implement eln style writing for reader.ZipStrategy --- .../ro_crate/preview/CratePreview.java | 75 ++++++++++++++- .../ro_crate/writer/ElnFormatWriter.java | 23 +++++ .../ro_crate/writer/FolderStrategy.java | 17 +++- .../ro_crate/writer/ZipStrategy.java | 96 ++++++++++++++++--- 4 files changed, 196 insertions(+), 15 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index 14ea76ea..46255020 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -2,8 +2,14 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; +import edu.kit.datamanager.ro_crate.writer.FolderStrategy; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import org.apache.commons.io.FileUtils; /** * Interface for the ROCrate preview. This manages the human-readable @@ -15,10 +21,77 @@ */ public interface CratePreview { + /** + * Generate a preview of the crate and store it into the given target directory. + * It is the caller's responsibility to handle, e.g. delete after use, the result + * (The caller takes ownership of the result). + *

+ * IMPORTANT NOTE: This method currently has a default implementation that relies + * on deprecated methods. In future, you will have to implement this method directly. + * + * @param crate the crate to generate a preview for. + * @param targetDir the target directory to store the preview in, + * owned by the caller. + * @throws IOException if an error occurs while generating the preview. + */ + default void generate(Crate crate, File targetDir) throws IOException { + // disable preview generation to avoid recursion, + // as this is usually called in the process of writing a crate + // (including preview) + new CrateWriter<>(new FolderStrategy().disablePreview()) + .save(crate, targetDir.getAbsolutePath()); + this.saveAllToFolder(targetDir); + try (var stream = Files.list(targetDir.toPath())) { + stream + .filter(path -> !path.getFileName().toString().equals("ro-crate-preview.html")) + .filter(path -> !path.getFileName().toString().equals("ro-crate-preview_files")) + .forEach(path -> { + try { + if (Files.isDirectory(path)) { + FileUtils.deleteDirectory(path.toFile()); + } else { + Files.delete(path); + } + } catch (IOException e) { + // Silently ignore deletion errors + } + }); + } + } + + /** + * Takes a crate in form of a zip file and generates a preview of it, + * which will be stored within the crate. + * + * @param zipFile the zip file with the crate, which should receive a preview. + * @throws IOException if an error occurs while saving the preview + * + * @deprecated Use {@link #generate(Crate, File)} instead. + */ + @Deprecated(since = "2.1.0", forRemoval = true) void saveAllToZip(ZipFile zipFile) throws IOException; + /** + * Saves the preview, given by the folder, into the given folder. + * + * @param folder the folder (containing a crate) to save the preview in. + * @throws IOException if an error occurs while saving the preview. + * + * @deprecated Use {@link #generate(Crate, File)} instead. + */ + @Deprecated(since = "2.1.0", forRemoval = true) void saveAllToFolder(File folder) throws IOException; - + + /** + * Saves the preview, given by the metadata, into the given stream. + * + * @param metadata the metadata of the crate to save the preview in. + * @param stream the stream to save the preview in. + * @throws IOException if an error occurs while saving the preview. + * + * @deprecated Use {@link #generate(Crate, File)} instead. + */ + @Deprecated(since = "2.1.0", forRemoval = true) void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java new file mode 100644 index 00000000..a808acef --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -0,0 +1,23 @@ +package edu.kit.datamanager.ro_crate.writer; + +import java.io.IOException; + +public interface ElnFormatWriter extends GenericWriterStrategy { + + /** + * Write in ELN format style, meaning with a root subfolder in the zip file. + * Same as {@link #withRootSubdirectory()}. + * + * @throws IOException if an error occurs + */ + ElnFormatWriter usingElnStyle(); + + /** + * Alias with more generic name for {@link #usingElnStyle()}. + * + * @throws IOException if an error occurs + */ + default ElnFormatWriter withRootSubdirectory() { + return this.usingElnStyle(); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index 8fa5a6bb..96a03de5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -25,6 +25,21 @@ public class FolderStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); + protected boolean writePreview = true; + + /** + * For internal use. Skips the preview generation when writing the crate. + * + * @return this instance of FolderStrategy + * + * @deprecated May be removed in future versions. Not intended for public use. + */ + @Deprecated(since = "2.1.0", forRemoval = true) + public FolderStrategy disablePreview() { + this.writePreview = false; + return this; + } + @Override public void save(Crate crate, String destination) throws IOException { File file = new File(destination); @@ -38,7 +53,7 @@ public void save(Crate crate, String destination) throws IOException { FileUtils.copyInputStreamToFile(inputStream, json); inputStream.close(); // save also the preview files to the crate destination - if (crate.getPreview() != null) { + if (crate.getPreview() != null && this.writePreview) { crate.getPreview().saveAllToFolder(file); } for (var e : crate.getUntrackedFiles()) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index 0c8634b1..b6452a9a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -5,42 +5,78 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import edu.kit.datamanager.ro_crate.preview.CratePreview; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.model.ZipParameters; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Matcher; /** * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStrategy implements GenericWriterStrategy { - +public class ZipStrategy implements + GenericWriterStrategy, + ElnFormatWriter +{ private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); + /** + * Defines if the zip file will directly contain the crate, + * or if it will contain a subdirectory with the crate. + */ + protected boolean createRootSubdir = false; + + @Override + public ElnFormatWriter usingElnStyle() { + this.createRootSubdir = true; + return this; + } + @Override public void save(Crate crate, String destination) throws IOException { + String innerFolderName = ""; + if (this.createRootSubdir) { + String dot = Matcher.quoteReplacement("."); + String end = Matcher.quoteReplacement("$"); + innerFolderName = Path.of(destination).getFileName() + .toString() + // remove .zip or .eln from the end of the file name + // (?i) removes case sensitivity + .replaceFirst("(?i)" + dot + "zip" + end, "") + .replaceFirst("(?i)" + dot + "eln" + end, ""); + if (!innerFolderName.endsWith("/")) { + innerFolderName += "/"; + } + } try (ZipFile zipFile = new ZipFile(destination)) { - saveMetadataJson(crate, zipFile); - saveDataEntities(crate, zipFile); + saveMetadataJson(crate, zipFile, innerFolderName); + saveDataEntities(crate, zipFile, innerFolderName); + savePreview(crate, zipFile, innerFolderName); } } - private void saveDataEntities(Crate crate, ZipFile zipFile) throws IOException { + private void saveDataEntities(Crate crate, ZipFile zipFile, String prefix) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - this.saveToZip(dataEntity, zipFile); + this.saveToZip(dataEntity, zipFile, prefix); } } - private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { + private void saveMetadataJson(Crate crate, ZipFile zipFile, String prefix) throws IOException { // write the metadata.json file ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); + zipParameters.setFileNameInZip(prefix + "ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); // we create an JsonNode only to have the file written pretty JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); @@ -49,12 +85,46 @@ private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { // write the ro-crate-metadata zipFile.addStream(inputStream, zipParameters); } - if (crate.getPreview() != null) { - crate.getPreview().saveAllToZip(zipFile); + } + + private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOException { + Optional preview = Optional.ofNullable(crate.getPreview()); + if (preview.isEmpty()) { + return; + } + final String ID = UUID.randomUUID().toString(); + File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + .resolve(ID) + .toFile(); + FileUtils.forceMkdir(tmpPreviewFolder); + FileUtils.forceDeleteOnExit(tmpPreviewFolder); + + preview.get().generate(crate, tmpPreviewFolder); + String[] paths = tmpPreviewFolder.list(); + if (paths == null) { + throw new IOException("No files found in temporary folder"); + } + for (String path : paths) { + File file = tmpPreviewFolder.toPath().resolve(path).toFile(); + if (file.isDirectory()) { + ZipParameters parameters = new ZipParameters(); + parameters.setRootFolderNameInZip(prefix + path); + parameters.setIncludeRootFolder(false); + zipFile.addFolder(file, parameters); + } else { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(prefix + path); + zipFile.addFile(file, zipParameters); + } + } + try { + FileUtils.forceDelete(tmpPreviewFolder); + } catch (IOException e) { + logger.error("Could not delete temporary preview folder: {}", tmpPreviewFolder); } } - private void saveToZip(DataEntity entity, ZipFile zipFile) throws IOException { + private void saveToZip(DataEntity entity, ZipFile zipFile, String prefix) throws IOException { if (entity == null || entity.getPath() == null) { return; } @@ -62,12 +132,12 @@ private void saveToZip(DataEntity entity, ZipFile zipFile) throws IOException { boolean isDirectory = entity.getPath().toFile().isDirectory(); if (isDirectory) { ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(entity.getId()); + parameters.setRootFolderNameInZip(prefix + entity.getId()); parameters.setIncludeRootFolder(false); zipFile.addFolder(entity.getPath().toFile(), parameters); } else { ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(entity.getId()); + zipParameters.setFileNameInZip(prefix + entity.getId()); zipFile.addFile(entity.getPath().toFile(), zipParameters); } } From a296f91aec08c73014981813e31643903ef33f0a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 18:52:10 +0200 Subject: [PATCH 53/79] test: add test for withRootSubdirectory --- .../ro_crate/reader/CommonReaderTest.java | 2 +- .../ro_crate/writer/ZipWriterTest.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java index 6cdc6fa1..1e162a1c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java @@ -27,7 +27,7 @@ * This parameter is only required to satisfy the generic reader strategy. * @param the type of the reader strategy */ -interface CommonReaderTest< +public interface CommonReaderTest< SOURCE_T, READER_STRATEGY extends GenericReaderStrategy > diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 80aa91ca..d9594b25 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -4,6 +4,13 @@ import java.nio.file.Path; import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.reader.CommonReaderTest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertTrue; class ZipWriterTest implements CommonWriterTest, @@ -20,4 +27,17 @@ public void saveCrateElnStyle(Crate crate, Path target) throws IOException { new CrateWriter<>(new ZipStrategy().usingElnStyle()) .save(crate, target.toAbsolutePath().toString()); } + + @Test + public void testAlias(@TempDir Path tmpDir) throws IOException { + Path zip = tmpDir.resolve("test.eln").toAbsolutePath(); + RoCrate crate = CommonReaderTest.newBaseCrate().build(); + + new CrateWriter<>(new ZipStrategy().withRootSubdirectory()) + .save(crate, zip.toString()); + + assertTrue(zip.toFile().exists(), "The zip file should exist"); + Path extractedPath = tmpDir.resolve("extracted"); + ensureCrateIsExtractedIn(zip, extractedPath); + } } From b25d1eb0c51210d15bfe79b34d93a09a6066b43e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 18:53:43 +0200 Subject: [PATCH 54/79] fix: remove IOException documentation from usingElnStyle and withRootSubdirectory methods --- .../kit/datamanager/ro_crate/writer/ElnFormatWriter.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java index a808acef..40cbf547 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -1,21 +1,15 @@ package edu.kit.datamanager.ro_crate.writer; -import java.io.IOException; - public interface ElnFormatWriter extends GenericWriterStrategy { /** * Write in ELN format style, meaning with a root subfolder in the zip file. * Same as {@link #withRootSubdirectory()}. - * - * @throws IOException if an error occurs */ ElnFormatWriter usingElnStyle(); /** * Alias with more generic name for {@link #usingElnStyle()}. - * - * @throws IOException if an error occurs */ default ElnFormatWriter withRootSubdirectory() { return this.usingElnStyle(); From a09d6f012c3eaf83116048a2c6549bc1d32fdcd7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 19:08:46 +0200 Subject: [PATCH 55/79] docs: add return documentation for usingElnStyle and withRootSubdirectory methods --- .../edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java index 40cbf547..55f09574 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -5,11 +5,15 @@ public interface ElnFormatWriter extends GenericWriterStrategy usingElnStyle(); /** * Alias with more generic name for {@link #usingElnStyle()}. + * + * @return this writer */ default ElnFormatWriter withRootSubdirectory() { return this.usingElnStyle(); From 00ee66e4035a4df28d9d7257f1eaa9be53c303a2 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 12 May 2025 16:03:26 +0200 Subject: [PATCH 56/79] test: share alias test with all eln writers and implement test for ZipStreamStrategyTest --- ...FormatTest.java => ElnFileWriterTest.java} | 24 ++++++++++++++++++- .../writer/ZipStreamStrategyTest.java | 18 +++++++++++++- .../ro_crate/writer/ZipWriterTest.java | 15 ++++-------- 3 files changed, 44 insertions(+), 13 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{ElnFileFormatTest.java => ElnFileWriterTest.java} (73%) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileWriterTest.java similarity index 73% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileWriterTest.java index feb8dc06..971822e1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileWriterTest.java @@ -4,6 +4,7 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.reader.CommonReaderTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -12,10 +13,11 @@ import static org.junit.jupiter.api.Assertions.*; -public interface ElnFileFormatTest extends TestableWriterStrategy { +public interface ElnFileWriterTest extends TestableWriterStrategy { /** * Write in ELN format style, meaning with a subfolder in the zip file. + * Must use {@link ElnFormatWriter#usingElnStyle()}. * * @param crate the crate to write * @param target the target path to the save location @@ -23,6 +25,14 @@ public interface ElnFileFormatTest extends TestableWriterStrategy { */ void saveCrateElnStyle(Crate crate, Path target) throws IOException; + /** + * Same as {@link #saveCrateElnStyle(Crate, Path)} but with the alias + * {@link ElnFormatWriter#withRootSubdirectory()}. + * @param crate the crate to write + * @param target the target path to the save location + */ + void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException; + @Test default void testMakesElnStyleCrate(@TempDir Path tempDir) throws IOException { // We need a correct directory to compare with. @@ -67,4 +77,16 @@ default void testMakesElnStyleCrate(@TempDir Path tempDir) throws IOException { builtCrate, "/json/crate/fileAndDir.json"); } + + @Test + default void testAlias(@TempDir Path tmpDir) throws IOException { + Path zip = tmpDir.resolve("test.eln").toAbsolutePath(); + RoCrate crate = CommonReaderTest.newBaseCrate().build(); + + this.saveCrateSubdirectoryStyle(crate, zip); + + assertTrue(zip.toFile().exists(), "The zip file should exist"); + Path extractedPath = tmpDir.resolve("extracted"); + ensureCrateIsExtractedIn(zip, extractedPath); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index aa3ee5c6..ac113667 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -4,11 +4,15 @@ import java.nio.file.Path; import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.RoCrate; /** * @author jejkal */ -class ZipStreamStrategyTest implements CommonWriterTest { +class ZipStreamStrategyTest implements + CommonWriterTest, + ElnFileWriterTest +{ @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -16,4 +20,16 @@ public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, stream); } } + + @Override + public void saveCrateElnStyle(Crate crate, Path target) throws IOException { + new CrateWriter<>(new ZipStreamStrategy().usingElnStyle()) + .save(crate, target.toAbsolutePath().toString()); + } + + @Override + public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { + new CrateWriter<>(new ZipStreamStrategy().withRootSubdirectory()) + .save(crate, target.toString()); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index d9594b25..3be1b2b3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -14,7 +14,7 @@ class ZipWriterTest implements CommonWriterTest, - ElnFileFormatTest + ElnFileWriterTest { @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -28,16 +28,9 @@ public void saveCrateElnStyle(Crate crate, Path target) throws IOException { .save(crate, target.toAbsolutePath().toString()); } - @Test - public void testAlias(@TempDir Path tmpDir) throws IOException { - Path zip = tmpDir.resolve("test.eln").toAbsolutePath(); - RoCrate crate = CommonReaderTest.newBaseCrate().build(); - + @Override + public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { new CrateWriter<>(new ZipStrategy().withRootSubdirectory()) - .save(crate, zip.toString()); - - assertTrue(zip.toFile().exists(), "The zip file should exist"); - Path extractedPath = tmpDir.resolve("extracted"); - ensureCrateIsExtractedIn(zip, extractedPath); + .save(crate, target.toString()); } } From e2a4bf3f1dc9eac9a1dca92c0a4447b9db39ce11 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 12 May 2025 16:44:41 +0200 Subject: [PATCH 57/79] feat: support ELN format in ZipStreamStrategy --- .../ro_crate/writer/ZipStreamStrategy.java | 114 +++++++++++++++--- .../writer/ZipStreamStrategyTest.java | 4 +- 2 files changed, 99 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index b97b3c2a..12318d24 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -7,16 +7,18 @@ import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Matcher; +import edu.kit.datamanager.ro_crate.preview.CratePreview; import edu.kit.datamanager.ro_crate.util.ZipUtil; -import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,28 +26,72 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStreamStrategy implements GenericWriterStrategy { +public class ZipStreamStrategy implements + GenericWriterStrategy, + ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); + /** + * Defines if the zip file will directly contain the crate, + * or if it will contain a subdirectory with the crate. + */ + protected boolean createRootSubdir = false; + protected String rootSubdirName = "content"; + + @Override + public ElnFormatWriter usingElnStyle() { + this.createRootSubdir = true; + return this; + } + + /** + * Sets the name of a root subdirectory in the zip file. + * Implicitly also enables the creation of a root subdirectory. + * If used for ELN files, note the subdirectory name should be the same as the zip + * files name. + * + * @param name the name of the subdirectory + * @return this instance of ZipStreamStrategy + */ + public ZipStreamStrategy setSubdirectoryName(String name) { + this.rootSubdirName = name; + this.createRootSubdir = true; + return this; + } + @Override public void save(Crate crate, OutputStream destination) throws IOException { + String innerFolderName = ""; + if (this.createRootSubdir) { + String dot = Matcher.quoteReplacement("."); + String end = Matcher.quoteReplacement("$"); + innerFolderName = this.rootSubdirName + // remove .zip or .eln from the end of the file name + // (?i) removes case sensitivity + .replaceFirst("(?i)" + dot + "zip" + end, "") + .replaceFirst("(?i)" + dot + "eln" + end, ""); + if (!innerFolderName.endsWith("/")) { + innerFolderName += "/"; + } + } try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { - saveMetadataJson(crate, zipFile); - saveDataEntities(crate, zipFile); + saveMetadataJson(crate, zipFile, innerFolderName); + saveDataEntities(crate, zipFile, innerFolderName); + savePreview(crate, zipFile, innerFolderName); } } - private void saveDataEntities(Crate crate, ZipOutputStream zipStream) throws IOException { + private void saveDataEntities(Crate crate, ZipOutputStream zipStream, String prefix) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - saveToStream(dataEntity, zipStream); + this.saveToStream(dataEntity, zipStream, prefix); } } - private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOException { + private void saveMetadataJson(Crate crate, ZipOutputStream zipStream, String prefix) throws IOException { // write the metadata.json file ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); + zipParameters.setFileNameInZip(prefix + "ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); // we create an JsonNode only to have the file written pretty JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); @@ -61,13 +107,47 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOE } } zipStream.closeEntry(); + } + + private void savePreview(Crate crate, ZipOutputStream zipStream, String prefix) throws IOException { + Optional preview = Optional.ofNullable(crate.getPreview()); + if (preview.isEmpty()) { + return; + } + final String ID = UUID.randomUUID().toString(); + File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + .resolve(ID) + .toFile(); + FileUtils.forceMkdir(tmpPreviewFolder); + FileUtils.forceDeleteOnExit(tmpPreviewFolder); - if (crate.getPreview() != null) { - crate.getPreview().saveAllToStream(str, zipStream); + preview.get().generate(crate, tmpPreviewFolder); + String[] paths = tmpPreviewFolder.list(); + if (paths == null) { + throw new IOException("No files found in temporary folder"); + } + for (String path : paths) { + File file = tmpPreviewFolder.toPath().resolve(path).toFile(); + if (file.isDirectory()) { + ZipUtil.addFolderToZipStream( + zipStream, + file, + prefix + path); + } else { + ZipUtil.addFileToZipStream( + zipStream, + file, + prefix + path); + } + } + try { + FileUtils.forceDelete(tmpPreviewFolder); + } catch (IOException e) { + logger.error("Could not delete temporary preview folder: {}", tmpPreviewFolder); } } - private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws IOException { + private void saveToStream(DataEntity entity, ZipOutputStream zipStream, String prefix) throws IOException { if (entity == null) { return; } @@ -77,12 +157,12 @@ private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws I ZipUtil.addFolderToZipStream( zipStream, entity.getPath().toAbsolutePath().toString(), - entity.getId()); + prefix + entity.getId()); } else { ZipUtil.addFileToZipStream( zipStream, entity.getPath().toFile(), - entity.getId()); + prefix + entity.getId()); } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index ac113667..c39cebe3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -24,12 +24,12 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { new CrateWriter<>(new ZipStreamStrategy().usingElnStyle()) - .save(crate, target.toAbsolutePath().toString()); + .save(crate, new FileOutputStream(target.toFile())); } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { new CrateWriter<>(new ZipStreamStrategy().withRootSubdirectory()) - .save(crate, target.toString()); + .save(crate, new FileOutputStream(target.toFile())); } } From 8455f4f9f121db8ffbc594762fc2d569528d7778 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 13:56:55 +0200 Subject: [PATCH 58/79] test: add tests for saving previews to ZipOutputStream in PreviewTest --- .../ro_crate/preview/PreviewTest.java | 109 +++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java index 2db987b3..0c1e67b0 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java @@ -1,13 +1,14 @@ package edu.kit.datamanager.ro_crate.preview; +import gg.jte.output.FileOutput; import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -75,6 +76,41 @@ void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); } + @Test + void staticPreviewSaveToZipStream(@TempDir Path dir) throws IOException { + var file1 = dir.resolve("file.html"); + FileUtils.writeStringToFile(file1.toFile(), "random html, does not need to be valid for this test", Charset.defaultCharset()); + + var file2 = dir.resolve("directory"); + var fileInDir = file2.resolve("fileInDir.html"); + FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); + StaticPreview preview = new StaticPreview(file1.toFile(), file2.toFile()); + + ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(dir.resolve("destination.zip").toFile())); + preview.saveAllToStream( + null, // static preview does not need metadata + stream); + stream.flush(); + stream.close(); + + try (ZipFile zf = new ZipFile(dir.resolve("destination.zip").toFile())) { + zf.extractAll(dir.resolve("extracted").toAbsolutePath().toString()); + } + + var e = dir.resolve("extracted"); + var roPreview = e.resolve("ro-crate-preview.html"); + var roDir = e.resolve("ro-crate-preview_files"); + var roDirFile = roDir.resolve("fileInDir.html"); + assertTrue(Files.isRegularFile(roPreview)); + assertTrue(Files.isDirectory(roDir)); + assertTrue(Files.isRegularFile(roDirFile)); + + assertTrue(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), file1.toFile(), String.valueOf(Charset.defaultCharset()))); + assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + + assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + } + @Test void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); @@ -116,6 +152,41 @@ void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } + @Test + void testAutomaticPreviewZipStream(@TempDir Path dir) throws IOException { + AutomaticPreview preview = new AutomaticPreview(); + String metadataPath = "/crates/other/idrc_project/ro-crate-metadata.json"; + Path crate = dir.resolve("crate"); + + File zipFile = dir.resolve("test.zip").toFile(); + { + ZipFile zip = new ZipFile(zipFile); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath); + zip.addStream(crateJson, zipParameters); + crateJson.close(); + } + String metadata = new String( + PreviewTest.class.getResourceAsStream(metadataPath) + .readAllBytes()); + ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile)); + preview.saveAllToStream(metadata, stream); + stream.flush(); + stream.close(); + + try { + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); + preview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid ZIP file for preview."); + } catch (IOException ex) { + //ok + } + new ZipFile(zipFile).extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } + @Test void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { CustomPreview customPreview = new CustomPreview(); @@ -167,4 +238,38 @@ void testCustomPreviewZip(@TempDir Path tmp) throws IOException { assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } + @Test + void testCustomPreviewZipStream(@TempDir Path tmp) throws IOException { + CustomPreview preview = new CustomPreview(); + String metadataPath = "/crates/other/idrc_project/ro-crate-metadata.json"; + Path crate = tmp.resolve("crate"); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + + File zipFile = tmp.resolve("test.zip").toFile(); + { + ZipFile zip = new ZipFile(zipFile); + InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath); + zip.addStream(crateJson, zipParameters); + crateJson.close(); + } + String metadata = new String( + PreviewTest.class.getResourceAsStream(metadataPath) + .readAllBytes()); + ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile)); + preview.saveAllToStream(metadata, stream); + stream.flush(); + stream.close(); + + try { + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(tmp.resolve("dddd.zip").toFile()); + preview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid input to preview."); + } catch (IOException ex) { + //ok + } + new ZipFile(zipFile).extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } } From 5ef25d400e71c0799177a97465193a09cc8af3c7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:05:39 +0200 Subject: [PATCH 59/79] fix: enforce correct subfolder name for additional files in StaticPreview.saveAllToStream --- .../edu/kit/datamanager/ro_crate/preview/StaticPreview.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index 5a627e17..cf81aa15 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -66,7 +66,7 @@ public void saveAllToFolder(File folder) throws IOException { public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { ZipUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); if (this.otherFiles != null) { - ZipUtil.addFolderToZipStream(stream, this.otherFiles, this.otherFiles.getName()); + ZipUtil.addFolderToZipStream(stream, this.otherFiles, "ro-crate-preview_files"); } } } From cfa6f73ebaf38aa6f77cdd0a279188b22629124e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:28:40 +0200 Subject: [PATCH 60/79] refactor: prefix reader strategies and writer strategies with Read or Write accordingly. This eases importing reader and writer strategies of the same kind (source/destination type). --- .../ro_crate/preview/CratePreview.java | 4 ++-- .../ro_crate/reader/FolderReader.java | 4 ++-- ...rStrategy.java => ReadFolderStrategy.java} | 2 +- ...{ZipStrategy.java => ReadZipStrategy.java} | 6 +++--- ...rategy.java => ReadZipStreamStrategy.java} | 8 ++++---- .../datamanager/ro_crate/reader/Readers.java | 20 +++++++++---------- .../ro_crate/reader/ZipReader.java | 4 ++-- .../ro_crate/writer/FolderWriter.java | 4 ++-- ...Strategy.java => WriteFolderStrategy.java} | 8 ++++---- ...ZipStrategy.java => WriteZipStrategy.java} | 4 ++-- ...ategy.java => WriteZipStreamStrategy.java} | 8 ++++---- .../datamanager/ro_crate/writer/Writers.java | 6 +++--- .../ro_crate/writer/ZipWriter.java | 4 ++-- .../ro_crate/examples/LearnByExampleTest.java | 8 ++++---- .../ro_crate/reader/FolderReaderTest.java | 8 ++++---- .../ro_crate/reader/ZipReaderTest.java | 10 +++++----- .../ro_crate/reader/ZipStreamReaderTest.java | 10 +++++----- .../writer/ZipStreamStrategyTest.java | 4 ++-- .../ro_crate/writer/ZipWriterTest.java | 8 ++------ 19 files changed, 63 insertions(+), 67 deletions(-) rename src/main/java/edu/kit/datamanager/ro_crate/reader/{FolderStrategy.java => ReadFolderStrategy.java} (92%) rename src/main/java/edu/kit/datamanager/ro_crate/reader/{ZipStrategy.java => ReadZipStrategy.java} (96%) rename src/main/java/edu/kit/datamanager/ro_crate/reader/{ZipStreamStrategy.java => ReadZipStreamStrategy.java} (96%) rename src/main/java/edu/kit/datamanager/ro_crate/writer/{FolderStrategy.java => WriteFolderStrategy.java} (93%) rename src/main/java/edu/kit/datamanager/ro_crate/writer/{ZipStrategy.java => WriteZipStrategy.java} (98%) rename src/main/java/edu/kit/datamanager/ro_crate/writer/{ZipStreamStrategy.java => WriteZipStreamStrategy.java} (96%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index 46255020..a69b47aa 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -6,7 +6,7 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.writer.CrateWriter; -import edu.kit.datamanager.ro_crate.writer.FolderStrategy; +import edu.kit.datamanager.ro_crate.writer.WriteFolderStrategy; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; @@ -38,7 +38,7 @@ default void generate(Crate crate, File targetDir) throws IOException { // disable preview generation to avoid recursion, // as this is usually called in the process of writing a crate // (including preview) - new CrateWriter<>(new FolderStrategy().disablePreview()) + new CrateWriter<>(new WriteFolderStrategy().disablePreview()) .save(crate, targetDir.getAbsolutePath()); this.saveAllToFolder(targetDir); try (var stream = Files.list(targetDir.toPath())) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java index f0436e27..c6268ac1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java @@ -6,7 +6,7 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 * - * @deprecated Use {@link FolderStrategy} instead. + * @deprecated Use {@link ReadFolderStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class FolderReader extends FolderStrategy {} +public class FolderReader extends ReadFolderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadFolderStrategy.java similarity index 92% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ReadFolderStrategy.java index 4b07e01d..927d77de 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadFolderStrategy.java @@ -14,7 +14,7 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -public class FolderStrategy implements GenericReaderStrategy { +public class ReadFolderStrategy implements GenericReaderStrategy { @Override public ObjectNode readMetadataJson(String location) throws IOException { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java index d25a235e..12200616 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java @@ -29,7 +29,7 @@ * persistent location and possibly read it from there, if required. Or use * the ZipWriter to write it back to its source. */ -public class ZipStrategy implements GenericReaderStrategy { +public class ReadZipStrategy implements GenericReaderStrategy { protected final String ID = UUID.randomUUID().toString(); protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipReader/%s/", ID)); @@ -38,7 +38,7 @@ public class ZipStrategy implements GenericReaderStrategy { /** * Crates a ZipReader with the default configuration as described in the class documentation. */ - public ZipStrategy() {} + public ReadZipStrategy() {} /** * Creates a ZipReader which will extract the contents temporary @@ -51,7 +51,7 @@ public ZipStrategy() {} * directory. These subdirectories * will have UUIDs as their names. */ - public ZipStrategy(Path folderPath, boolean shallAddUuidSubfolder) { + public ReadZipStrategy(Path folderPath, boolean shallAddUuidSubfolder) { if (shallAddUuidSubfolder) { this.temporaryFolder = folderPath.resolve(ID); } else { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index f6ed4577..60f908a8 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -25,9 +25,9 @@ * * @author jejkal */ -public class ZipStreamStrategy implements GenericReaderStrategy { +public class ReadZipStreamStrategy implements GenericReaderStrategy { - private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(ReadZipStreamStrategy.class); protected final String ID = UUID.randomUUID().toString(); protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipStreamReader/%s/", ID)); protected boolean isExtracted = false; @@ -36,7 +36,7 @@ public class ZipStreamStrategy implements GenericReaderStrategy { * Crates a ZipStreamReader with the default configuration as described in * the class documentation. */ - public ZipStreamStrategy() { + public ReadZipStreamStrategy() { } /** @@ -49,7 +49,7 @@ public ZipStreamStrategy() { * subdirectories of the given directory. These subdirectories will have * UUIDs as their names. */ - public ZipStreamStrategy(Path folderPath, boolean shallAddUuidSubfolder) { + public ReadZipStreamStrategy(Path folderPath, boolean shallAddUuidSubfolder) { if (shallAddUuidSubfolder) { this.temporaryFolder = folderPath.resolve(ID); } else { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java index 49e09cb1..83a43701 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java @@ -19,10 +19,10 @@ private Readers() {} * * @return A reader configured for ZIP files * - * @see ZipStreamStrategy#ZipStreamStrategy() + * @see ReadZipStreamStrategy#ReadZipStreamStrategy() */ public static CrateReader newZipStreamReader() { - return new CrateReader<>(new ZipStreamStrategy()); + return new CrateReader<>(new ReadZipStreamStrategy()); } /** @@ -33,10 +33,10 @@ public static CrateReader newZipStreamReader() { * @param useUuidSubfolder Whether to create a UUID subfolder under extractPath * @return A reader configured for ZIP files with custom extraction * - * @see ZipStreamStrategy#ZipStreamStrategy(Path, boolean) + * @see ReadZipStreamStrategy#ReadZipStreamStrategy(Path, boolean) */ public static CrateReader newZipStreamReader(Path extractPath, boolean useUuidSubfolder) { - return new CrateReader<>(new ZipStreamStrategy(extractPath, useUuidSubfolder)); + return new CrateReader<>(new ReadZipStreamStrategy(extractPath, useUuidSubfolder)); } /** @@ -44,10 +44,10 @@ public static CrateReader newZipStreamReader(Path extractPath, bool * * @return A reader configured for folders * - * @see FolderStrategy + * @see ReadFolderStrategy */ public static CrateReader newFolderReader() { - return new CrateReader<>(new FolderStrategy()); + return new CrateReader<>(new ReadFolderStrategy()); } /** @@ -55,10 +55,10 @@ public static CrateReader newFolderReader() { * * @return A reader configured for ZIP files * - * @see ZipStrategy#ZipStrategy() + * @see ReadZipStrategy#ReadZipStrategy() */ public static CrateReader newZipPathReader() { - return new CrateReader<>(new ZipStrategy()); + return new CrateReader<>(new ReadZipStrategy()); } /** @@ -69,9 +69,9 @@ public static CrateReader newZipPathReader() { * @param useUuidSubfolder Whether to create a UUID subfolder under extractPath * @return A reader configured for ZIP files with custom extraction * - * @see ZipStrategy#ZipStrategy(Path, boolean) + * @see ReadZipStrategy#ReadZipStrategy(Path, boolean) */ public static CrateReader newZipPathReader(Path extractPath, boolean useUuidSubfolder) { - return new CrateReader<>(new ZipStrategy(extractPath, useUuidSubfolder)); + return new CrateReader<>(new ReadZipStrategy(extractPath, useUuidSubfolder)); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java index ad9bbb01..d92faf94 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java @@ -18,10 +18,10 @@ * persistent location and possibly read it from there, if required. Or use * the ZipWriter to write it back to its source. * - * @deprecated Use {@link ZipStrategy} instead. + * @deprecated Use {@link ReadZipStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class ZipReader extends ZipStrategy { +public class ZipReader extends ReadZipStrategy { /** * Crates a ZipReader with the default configuration as described in the class documentation. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java index 5730104e..1426414e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java @@ -5,7 +5,7 @@ * * @author Nikola Tzotchev on 9.2.2022 г. * - * @deprecated Use {@link FolderStrategy} instead. + * @deprecated Use {@link WriteFolderStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class FolderWriter extends FolderStrategy {} +public class FolderWriter extends WriteFolderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java similarity index 93% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java index 96a03de5..8a4844ed 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java @@ -21,21 +21,21 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -public class FolderStrategy implements GenericWriterStrategy { +public class WriteFolderStrategy implements GenericWriterStrategy { - private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(WriteFolderStrategy.class); protected boolean writePreview = true; /** * For internal use. Skips the preview generation when writing the crate. * - * @return this instance of FolderStrategy + * @return this instance of WriteFolderStrategy * * @deprecated May be removed in future versions. Not intended for public use. */ @Deprecated(since = "2.1.0", forRemoval = true) - public FolderStrategy disablePreview() { + public WriteFolderStrategy disablePreview() { this.writePreview = false; return this; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java similarity index 98% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java index b6452a9a..848f5240 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java @@ -26,11 +26,11 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStrategy implements +public class WriteZipStrategy implements GenericWriterStrategy, ElnFormatWriter { - private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(WriteZipStrategy.class); /** * Defines if the zip file will directly contain the crate, diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index 12318d24..c941bc89 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -26,11 +26,11 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStreamStrategy implements +public class WriteZipStreamStrategy implements GenericWriterStrategy, ElnFormatWriter { - private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(WriteZipStreamStrategy.class); /** * Defines if the zip file will directly contain the crate, @@ -52,9 +52,9 @@ public ElnFormatWriter usingElnStyle() { * files name. * * @param name the name of the subdirectory - * @return this instance of ZipStreamStrategy + * @return this instance of ReadZipStreamStrategy */ - public ZipStreamStrategy setSubdirectoryName(String name) { + public WriteZipStreamStrategy setSubdirectoryName(String name) { this.rootSubdirName = name; this.createRootSubdir = true; return this; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java index 5b691ece..d1a41d66 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java @@ -19,7 +19,7 @@ private Writers() {} * @return a new instance of {@link CrateWriter} for writing to a folder */ public static CrateWriter newFolderWriter() { - return new CrateWriter<>(new FolderStrategy()); + return new CrateWriter<>(new WriteFolderStrategy()); } /** @@ -28,7 +28,7 @@ public static CrateWriter newFolderWriter() { * @return a new instance of {@link CrateWriter} for writing to a zip stream */ public static CrateWriter newZipStreamWriter() { - return new CrateWriter<>(new ZipStreamStrategy()); + return new CrateWriter<>(new WriteZipStreamStrategy()); } /** @@ -37,6 +37,6 @@ public static CrateWriter newZipStreamWriter() { * @return a new instance of {@link CrateWriter} for writing to a zip file */ public static CrateWriter newZipPathWriter() { - return new CrateWriter<>(new ZipStrategy()); + return new CrateWriter<>(new WriteZipStrategy()); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java index f5261003..c9c0735e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java @@ -4,7 +4,7 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. * - * @deprecated Use {@link ZipStrategy} instead. + * @deprecated Use {@link WriteZipStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class ZipWriter extends ZipStrategy {} +public class ZipWriter extends WriteZipStrategy {} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java index 64733e89..89c5a8f8 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java @@ -17,7 +17,7 @@ import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; import edu.kit.datamanager.ro_crate.writer.CrateWriter; -import edu.kit.datamanager.ro_crate.writer.FolderStrategy; +import edu.kit.datamanager.ro_crate.writer.WriteFolderStrategy; import edu.kit.datamanager.ro_crate.writer.GenericWriterStrategy; import edu.kit.datamanager.ro_crate.writer.Writers; import org.apache.commons.io.FileUtils; @@ -336,13 +336,13 @@ void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { // Now, let's write it to a folder. Note the used strategy could be replaced with your own. Path folder = tempDir.resolve("folderCrate"); - new CrateWriter<>(new FolderStrategy()) + new CrateWriter<>(new WriteFolderStrategy()) .save(crate, folder.toString()); // and read it back. RoCrate read = new CrateReader<>( - // Note: There are two FolderStrategy implementations, one for reading and one for writing. + // Note: There are two WriteFolderStrategy implementations, one for reading and one for writing. // Java is a bit bad with imports, so we use the fully qualified name here. - new edu.kit.datamanager.ro_crate.reader.FolderStrategy() + new edu.kit.datamanager.ro_crate.reader.ReadFolderStrategy() ) .readCrate(folder.toAbsolutePath().toString()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index 8375ba36..83bf4f78 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -17,7 +17,7 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderReaderTest implements CommonReaderTest +class FolderReaderTest implements CommonReaderTest { @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -31,15 +31,15 @@ public Crate readCrate(Path source) throws IOException { } @Override - public FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public ReadFolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { // This strategy does not support a non-default temporary directory // and will always use the default one. // It also has no state we could make assertions on. - return new FolderStrategy(); + return new ReadFolderStrategy(); } @Override - public Crate readCrate(FolderStrategy strategy, Path source) throws IOException { + public Crate readCrate(ReadFolderStrategy strategy, Path source) throws IOException { return new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 7d71d94e..35b168e1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -9,8 +9,8 @@ import static org.junit.jupiter.api.Assertions.*; class ZipReaderTest implements - CommonReaderTest, - ElnFileFormatTest + CommonReaderTest, + ElnFileFormatTest { @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -24,8 +24,8 @@ public Crate readCrate(Path source) throws IOException { } @Override - public ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { - ZipStrategy strategy = new ZipStrategy(tmpDirectory, useUuidSubfolder); + public ReadZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + ReadZipStrategy strategy = new ReadZipStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { assertEquals(strategy.getTemporaryFolder().getFileName().toString(), strategy.getID()); @@ -37,7 +37,7 @@ public ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSu } @Override - public Crate readCrate(ZipStrategy strategy, Path source) throws IOException { + public Crate readCrate(ReadZipStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); assertTrue(strategy.isExtracted()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index a6cca8e6..52feda9d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -11,8 +11,8 @@ class ZipStreamReaderTest implements - CommonReaderTest, - ElnFileFormatTest + CommonReaderTest, + ElnFileFormatTest { /** * At the point of writing this test, @@ -39,8 +39,8 @@ public Crate readCrate(Path source) throws IOException { } @Override - public ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { - ZipStreamStrategy strategy = new ZipStreamStrategy(tmpDirectory, useUuidSubfolder); + public ReadZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + ReadZipStreamStrategy strategy = new ReadZipStreamStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { assertEquals(strategy.getTemporaryFolder().getFileName().toString(), strategy.getID()); @@ -52,7 +52,7 @@ public ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean use } @Override - public Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { + public Crate readCrate(ReadZipStreamStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(new FileInputStream(source.toFile())); assertTrue(strategy.isExtracted()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index c39cebe3..eb9a927f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -23,13 +23,13 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStreamStrategy().usingElnStyle()) + new CrateWriter<>(new WriteZipStreamStrategy().usingElnStyle()) .save(crate, new FileOutputStream(target.toFile())); } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStreamStrategy().withRootSubdirectory()) + new CrateWriter<>(new WriteZipStreamStrategy().withRootSubdirectory()) .save(crate, new FileOutputStream(target.toFile())); } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 3be1b2b3..7e7a6b1d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -5,10 +5,6 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.reader.CommonReaderTest; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -24,13 +20,13 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStrategy().usingElnStyle()) + new CrateWriter<>(new WriteZipStrategy().usingElnStyle()) .save(crate, target.toAbsolutePath().toString()); } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStrategy().withRootSubdirectory()) + new CrateWriter<>(new WriteZipStrategy().withRootSubdirectory()) .save(crate, target.toString()); } } From 92ce1cb69da0f8936a7008bfea4c4dd0ce96cf1f Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:32:16 +0200 Subject: [PATCH 61/79] refactor: remove unused import in ZipWriterTest --- .../java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 7e7a6b1d..bfb29c3d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -6,8 +6,6 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.RoCrate; -import static org.junit.jupiter.api.Assertions.assertTrue; - class ZipWriterTest implements CommonWriterTest, ElnFileWriterTest From 87f790b10a43357fd2218c9ea86477d71e88f498 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:40:02 +0200 Subject: [PATCH 62/79] refactor: rename ZipStreamStrategyTest to ZipStreamWriterTest and update references --- .../datamanager/ro_crate/writer/WriteZipStreamStrategy.java | 5 +++++ .../datamanager/ro_crate/writer/TestableWriterStrategy.java | 2 +- .../{ZipStreamStrategyTest.java => ZipStreamWriterTest.java} | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{ZipStreamStrategyTest.java => ZipStreamWriterTest.java} (96%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index c941bc89..16735219 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -37,6 +37,11 @@ public class WriteZipStreamStrategy implements * or if it will contain a subdirectory with the crate. */ protected boolean createRootSubdir = false; + + /** + * In streams, we do not have a file name yet (or do not know it), + * so we need to set a default name for the root subdirectory. + */ protected String rootSubdirName = "content"; @Override diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java index 6cc41a30..079fb766 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java @@ -51,7 +51,7 @@ default void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throw */ default void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { FileUtils.forceMkdir(correctCrate.toFile()); - InputStream fileJson = ZipStreamStrategyTest.class + InputStream fileJson = ZipStreamWriterTest.class .getResourceAsStream("/json/crate/fileAndDir.json"); Assertions.assertNotNull(fileJson); // fill the directory with expected files and dirs diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java similarity index 96% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java index eb9a927f..3f022cf8 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java @@ -9,7 +9,7 @@ /** * @author jejkal */ -class ZipStreamStrategyTest implements +class ZipStreamWriterTest implements CommonWriterTest, ElnFileWriterTest { From b22e70bd837d83edcaab4d56db77fcfea3417db9 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 15:23:03 +0200 Subject: [PATCH 63/79] docs: Improve reader and writer documentation, and related type parameter names --- .../reader/GenericReaderStrategy.java | 8 ++--- .../ro_crate/reader/ReadZipStrategy.java | 22 ++++++++++--- .../reader/ReadZipStreamStrategy.java | 33 +++++++++++++++---- .../ro_crate/writer/CrateWriter.java | 8 ++--- .../ro_crate/writer/ElnFormatWriter.java | 12 +++++-- .../writer/GenericWriterStrategy.java | 6 ++-- 6 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java index 03f6b9c9..6a9f4bd4 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java @@ -8,16 +8,16 @@ * Generic interface for the strategy of the reader class. * This allows for flexible input types when implementing different reading strategies. * - * @param the type of the location parameter + * @param the type which determines the source of the crate */ -public interface GenericReaderStrategy { +public interface GenericReaderStrategy { /** * Read the metadata.json file from the given location. * * @param location the location to read from * @return the parsed metadata.json as ObjectNode */ - ObjectNode readMetadataJson(T location) throws IOException; + ObjectNode readMetadataJson(SOURCE_TYPE location) throws IOException; /** * Read the content from the given location. @@ -25,5 +25,5 @@ public interface GenericReaderStrategy { * @param location the location to read from * @return the content as a File */ - File readContent(T location) throws IOException; + File readContent(SOURCE_TYPE location) throws IOException; } \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java index 12200616..dffbedc7 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java @@ -14,11 +14,20 @@ import java.util.UUID; /** - * A ReaderStrategy implementation which reads from ZipFiles. + * Reads a crate from a ZIP archive (file). *

- * May be used as a dependency for CrateReader. It will unzip - * the ZipFile in a path relative to the directory this application runs in. - * By default, it will be `./.tmp/ro-crate-java/zipReader/$UUID/`. + * This class handles reading and extraction of RO-Crate content from ZIP archives + * into a temporary directory structure on the file system, + * which allows accessing the contained files. + *

+ * Supports ELN-Style crates, + * meaning the crate may be either in the zip archive directly or in a single, + * direct subfolder beneath the root folder (/folder). + *

+ * Note: This implementation checks for up to 50 subdirectories if multiple are present. + * This is to avoid zip bombs, which may contain a lot of subdirectories, + * and at the same time gracefully handle valid crated with hidden subdirectories + * (for example, thumbnails). *

* NOTE: The resulting crate may refer to these temporary files. Therefore, * these files are only being deleted before the JVM exits. If you need to free @@ -36,7 +45,10 @@ public class ReadZipStrategy implements GenericReaderStrategy { protected boolean isExtracted = false; /** - * Crates a ZipReader with the default configuration as described in the class documentation. + * Crates an instance with the default configuration. + *

+ * The default configuration is to extract the ZipFile to + * `./.tmp/ro-crate-java/zipReader/$UUID/`. */ public ReadZipStrategy() {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index 60f908a8..a8213b90 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -19,9 +19,29 @@ import org.slf4j.LoggerFactory; /** - * A ZIP file reader implementation of the StreamReaderStrategy interface. + * Reads a crate from a streamed ZIP archive. + *

* This class handles reading and extraction of RO-Crate content from ZIP archives - * into a temporary directory structure, which allows for accessing the contained files. + * into a temporary directory structure on the file system, + * which allows accessing the contained files. + *

+ * Supports ELN-Style crates, + * meaning the crate may be either in the zip archive directly or in a single, + * direct subfolder beneath the root folder (/folder). + *

+ * Note: This implementation checks for up to 50 subdirectories if multiple are present. + * This is to avoid zip bombs, which may contain a lot of subdirectories, + * and at the same time gracefully handle valid crated with hidden subdirectories + * (for example, thumbnails). + *

+ * NOTE: The resulting crate may refer to these temporary files. Therefore, + * these files are only being deleted before the JVM exits. If you need to free + * space because your application is long-running or creates a lot of + * crates, you may use the getters to retrieve information which will help + * you to clean up manually. Keep in mind that crates may refer to this + * folder after extraction. Use RoCrateWriter to export it so some + * persistent location and possibly read it from there, if required. Or use + * the ZipWriter to write it back to its source. * * @author jejkal */ @@ -33,11 +53,12 @@ public class ReadZipStreamStrategy implements GenericReaderStrategy protected boolean isExtracted = false; /** - * Crates a ZipStreamReader with the default configuration as described in - * the class documentation. + * Crates an instance with the default configuration. + *

+ * The default configuration is to extract the ZipFile to + * `./.tmp/ro-crate-java/zipStreamReader/%UUID/`. */ - public ReadZipStreamStrategy() { - } + public ReadZipStreamStrategy() {} /** * Creates a ZipStreamReader which will extract the contents temporary to diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java index 4ee1be6b..970803ec 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -10,11 +10,11 @@ * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) */ -public class CrateWriter { +public class CrateWriter { - private final GenericWriterStrategy strategy; + private final GenericWriterStrategy strategy; - public CrateWriter(GenericWriterStrategy strategy) { + public CrateWriter(GenericWriterStrategy strategy) { this.strategy = strategy; } @@ -24,7 +24,7 @@ public CrateWriter(GenericWriterStrategy strategy) { * @param crate the crate to write. * @param destination the location where the crate should be written. */ - public void save(Crate crate, DESTINATION destination) throws IOException { + public void save(Crate crate, DESTINATION_TYPE destination) throws IOException { Validator defaultValidation = new Validator(new JsonSchemaValidation()); defaultValidation.validate(crate); this.strategy.save(crate, destination); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java index 55f09574..bd6310cd 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -1,6 +1,12 @@ package edu.kit.datamanager.ro_crate.writer; -public interface ElnFormatWriter extends GenericWriterStrategy { +/** + * An Interface for {@link GenericWriterStrategy} implementations which support writing + * ELN-Style crates. + * + * @param the type which determines the destination of the result + */ +public interface ElnFormatWriter extends GenericWriterStrategy { /** * Write in ELN format style, meaning with a root subfolder in the zip file. @@ -8,14 +14,14 @@ public interface ElnFormatWriter extends GenericWriterStrategy usingElnStyle(); + ElnFormatWriter usingElnStyle(); /** * Alias with more generic name for {@link #usingElnStyle()}. * * @return this writer */ - default ElnFormatWriter withRootSubdirectory() { + default ElnFormatWriter withRootSubdirectory() { return this.usingElnStyle(); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java index c0e79779..d06301a0 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java @@ -8,14 +8,14 @@ * Generic interface for the strategy of the writer class. * This allows for flexible output types when implementing different writing strategies. * - * @param the type of the destination parameter + * @param the type of the destination parameter */ -public interface GenericWriterStrategy { +public interface GenericWriterStrategy { /** * Saves the given crate to the specified destination. * * @param crate The crate to save * @param destination The destination where the crate should be saved */ - void save(Crate crate, DESTINATION destination) throws IOException; + void save(Crate crate, DESTINATION_TYPE destination) throws IOException; } From 896018de90f1557a80cb34d972b722313f78d779 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 16:23:32 +0200 Subject: [PATCH 64/79] refactor: started sharing code between WriteZipStrategy and WriteZipStreamStrategy. --- .../ro_crate/preview/AutomaticPreview.java | 12 ++--- .../ro_crate/preview/CustomPreview.java | 8 ++-- .../ro_crate/preview/StaticPreview.java | 6 +-- .../ro_crate/util/FileSystemUtil.java | 48 +++++++++++++++++++ .../util/{ZipUtil.java => ZipStreamUtil.java} | 2 +- .../ro_crate/writer/WriteZipStrategy.java | 23 ++++----- .../writer/WriteZipStreamStrategy.java | 32 ++++++------- 7 files changed, 85 insertions(+), 46 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java rename src/main/java/edu/kit/datamanager/ro_crate/util/{ZipUtil.java => ZipStreamUtil.java} (99%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index 38241476..c403ab1c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -1,6 +1,6 @@ package edu.kit.datamanager.ro_crate.preview; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -10,9 +10,9 @@ import org.apache.commons.io.FileUtils; /** - * The default preview should use the rochtml tool - * (https://www.npmjs.com/package/ro-crate-html-js) for creating a simple - * preview file. + * The default preview should use the + * rochtml tool + * for creating a simple preview file. * * @author Nikola Tzotchev on 6.2.2022 г. * @version 1 @@ -66,13 +66,13 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx if (PreviewGenerator.isRochtmlAvailable()) { try { FileUtils.forceMkdir(new File("temp")); - try (FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json"))) { + try (FileWriter writer = new FileWriter("temp/ro-crate-metadata.json")) { writer.write(metadata); writer.flush(); } if (PreviewGenerator.isRochtmlAvailable()) { PreviewGenerator.generatePreview("temp"); - ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + ZipStreamUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } } finally { try { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index 5c8c9fa2..77300a87 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; @@ -53,7 +53,7 @@ public CustomPreview() { private CustomPreviewModel mapFromJson(String metadata) throws IOException { ObjectMapper mapper = new ObjectMapper(); - JsonNode root = (JsonNode) mapper.readValue(metadata, JsonNode.class); + JsonNode root = mapper.readValue(metadata, JsonNode.class); JsonNode graph = root.get("@graph"); CustomPreviewModel.ROCrate crate = new CustomPreviewModel.ROCrate(); List datasets = new ArrayList<>(); @@ -196,13 +196,13 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx //prepare output folder and writer FileUtils.forceMkdir(new File("temp")); //load and process template - try (FileWriter writer = new FileWriter(new File("temp/ro-crate-preview.html"))) { + try (FileWriter writer = new FileWriter("temp/ro-crate-preview.html")) { //load and process template template.process(dataModel, writer); writer.flush(); } - ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + ZipStreamUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } catch (TemplateException ex) { throw new IOException("Failed to generate preview.", ex); } finally { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index cf81aa15..c3a93629 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -1,6 +1,6 @@ package edu.kit.datamanager.ro_crate.preview; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import java.io.File; import java.io.IOException; @@ -64,9 +64,9 @@ public void saveAllToFolder(File folder) throws IOException { @Override public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { - ZipUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); + ZipStreamUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); if (this.otherFiles != null) { - ZipUtil.addFolderToZipStream(stream, this.otherFiles, "ro-crate-preview_files"); + ZipStreamUtil.addFolderToZipStream(stream, this.otherFiles, "ro-crate-preview_files"); } } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java new file mode 100644 index 00000000..0a1f9e0f --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java @@ -0,0 +1,48 @@ +package edu.kit.datamanager.ro_crate.util; + +import java.util.Collection; +import java.util.regex.Matcher; + +public class FileSystemUtil { + private FileSystemUtil() { + // Utility class, no instantiation + } + + /** + * Removes a specific set of given file extensions from a file name, if present. + * The extensions are case-insensitive. Given "ELN", "eln" or "Eln" will also match. + * The dot (.) before the extension is also assumed and removed implicitly: + *

+ * Example: + * filterExtensionsFromFileName("test.eln", Set.of("ELN")) -> "test" + * + * @param filename the file name to filter + * @param extensionsToRemove the extensions to remove + * @return the filtered file name + */ + public static String filterExtensionsFromFileName(String filename, Collection extensionsToRemove) { + String dot = Matcher.quoteReplacement("."); + String end = Matcher.quoteReplacement("$"); + for (String extension : extensionsToRemove) { + // (?i) removes case sensitivity + filename = filename.replaceFirst("(?i)" + dot + extension + end, ""); + } + return filename; + } + + /** + * Ensures that a given path ends with a trailing slash. + * + * @param path the path to check + * @return the path with a trailing slash if it didn't have one, or the original path + */ + public static String ensureTrailingSlash(String path) { + if (path == null || path.isEmpty()) { + return path; + } + if (!path.endsWith("/")) { + return path + "/"; + } + return path; + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipStreamUtil.java similarity index 99% rename from src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java rename to src/main/java/edu/kit/datamanager/ro_crate/util/ZipStreamUtil.java index c1da0137..b8eb2ef1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipStreamUtil.java @@ -11,7 +11,7 @@ * * @author jejkal */ -public class ZipUtil { +public class ZipStreamUtil { /** * Adds a folder and its contents to a ZipOutputStream. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java index 848f5240..04812054 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java @@ -6,6 +6,7 @@ import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import edu.kit.datamanager.ro_crate.preview.CratePreview; +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -19,8 +20,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Optional; +import java.util.Set; import java.util.UUID; -import java.util.regex.Matcher; /** * Implementation of the writing strategy to provide a way of writing crates to @@ -31,6 +32,7 @@ public class WriteZipStrategy implements ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(WriteZipStrategy.class); + public static final String TMP_DIR = "./.tmp/ro-crate-java/writer-zip-strategy/"; /** * Defines if the zip file will directly contain the crate, @@ -48,17 +50,10 @@ public ElnFormatWriter usingElnStyle() { public void save(Crate crate, String destination) throws IOException { String innerFolderName = ""; if (this.createRootSubdir) { - String dot = Matcher.quoteReplacement("."); - String end = Matcher.quoteReplacement("$"); - innerFolderName = Path.of(destination).getFileName() - .toString() - // remove .zip or .eln from the end of the file name - // (?i) removes case sensitivity - .replaceFirst("(?i)" + dot + "zip" + end, "") - .replaceFirst("(?i)" + dot + "eln" + end, ""); - if (!innerFolderName.endsWith("/")) { - innerFolderName += "/"; - } + innerFolderName = FileSystemUtil.filterExtensionsFromFileName( + Path.of(destination).getFileName().toString(), + Set.of("ELN", "ZIP")); + innerFolderName = FileSystemUtil.ensureTrailingSlash(innerFolderName); } try (ZipFile zipFile = new ZipFile(destination)) { saveMetadataJson(crate, zipFile, innerFolderName); @@ -93,7 +88,7 @@ private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOE return; } final String ID = UUID.randomUUID().toString(); - File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + File tmpPreviewFolder = Path.of(TMP_DIR) .resolve(ID) .toFile(); FileUtils.forceMkdir(tmpPreviewFolder); @@ -102,7 +97,7 @@ private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOE preview.get().generate(crate, tmpPreviewFolder); String[] paths = tmpPreviewFolder.list(); if (paths == null) { - throw new IOException("No files found in temporary folder"); + throw new IOException("No preview files found in temporary folder. Preview generation failed."); } for (String path : paths) { File file = tmpPreviewFolder.toPath().resolve(path).toFile(); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index 16735219..8c54cc95 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -11,11 +11,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Optional; +import java.util.Set; import java.util.UUID; -import java.util.regex.Matcher; import edu.kit.datamanager.ro_crate.preview.CratePreview; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -31,6 +32,7 @@ public class WriteZipStreamStrategy implements ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(WriteZipStreamStrategy.class); + public static final String TMP_DIR = "./.tmp/ro-crate-java/writer-zip-stream-strategy/"; /** * Defines if the zip file will directly contain the crate, @@ -69,16 +71,10 @@ public WriteZipStreamStrategy setSubdirectoryName(String name) { public void save(Crate crate, OutputStream destination) throws IOException { String innerFolderName = ""; if (this.createRootSubdir) { - String dot = Matcher.quoteReplacement("."); - String end = Matcher.quoteReplacement("$"); - innerFolderName = this.rootSubdirName - // remove .zip or .eln from the end of the file name - // (?i) removes case sensitivity - .replaceFirst("(?i)" + dot + "zip" + end, "") - .replaceFirst("(?i)" + dot + "eln" + end, ""); - if (!innerFolderName.endsWith("/")) { - innerFolderName += "/"; - } + innerFolderName = FileSystemUtil.filterExtensionsFromFileName( + this.rootSubdirName, + Set.of("ELN", "ZIP")); + innerFolderName = FileSystemUtil.ensureTrailingSlash(innerFolderName); } try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { saveMetadataJson(crate, zipFile, innerFolderName); @@ -120,7 +116,7 @@ private void savePreview(Crate crate, ZipOutputStream zipStream, String prefix) return; } final String ID = UUID.randomUUID().toString(); - File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + File tmpPreviewFolder = Path.of(TMP_DIR) .resolve(ID) .toFile(); FileUtils.forceMkdir(tmpPreviewFolder); @@ -129,17 +125,17 @@ private void savePreview(Crate crate, ZipOutputStream zipStream, String prefix) preview.get().generate(crate, tmpPreviewFolder); String[] paths = tmpPreviewFolder.list(); if (paths == null) { - throw new IOException("No files found in temporary folder"); + throw new IOException("No preview files found in temporary folder. Preview generation failed."); } for (String path : paths) { File file = tmpPreviewFolder.toPath().resolve(path).toFile(); if (file.isDirectory()) { - ZipUtil.addFolderToZipStream( + ZipStreamUtil.addFolderToZipStream( zipStream, file, prefix + path); } else { - ZipUtil.addFileToZipStream( + ZipStreamUtil.addFileToZipStream( zipStream, file, prefix + path); @@ -159,12 +155,12 @@ private void saveToStream(DataEntity entity, ZipOutputStream zipStream, String p boolean isDirectory = entity.getPath().toFile().isDirectory(); if (isDirectory) { - ZipUtil.addFolderToZipStream( + ZipStreamUtil.addFolderToZipStream( zipStream, entity.getPath().toAbsolutePath().toString(), prefix + entity.getId()); } else { - ZipUtil.addFileToZipStream( + ZipStreamUtil.addFileToZipStream( zipStream, entity.getPath().toFile(), prefix + entity.getId()); From fd9b84db617f66b276e5b365724759da412b9416 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:11:48 +0200 Subject: [PATCH 65/79] refactor: factor out common tmp directory handling in zip readers --- .../ro_crate/reader/ReadZipStrategy.java | 8 ++--- .../reader/ReadZipStreamStrategy.java | 13 ++------ .../ro_crate/util/FileSystemUtil.java | 24 ++++++++++++++ .../ro_crate/util/FileSystemUtilTest.java | 31 +++++++++++++++++++ 4 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java index dffbedc7..5a2474b5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -94,12 +95,7 @@ public boolean isExtracted() { private void readCrate(String location) throws IOException { File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } + FileSystemUtil.mkdirOrDeleteContent(folder); // extract try (ZipFile zf = new ZipFile(location)) { zf.extractAll(temporaryFolder.toAbsolutePath().toString()); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index a8213b90..afaa217e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -11,6 +11,8 @@ import java.io.OutputStream; import java.nio.file.Path; import java.util.UUID; + +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import net.lingala.zip4j.io.inputstream.ZipInputStream; import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; @@ -108,16 +110,7 @@ public boolean isExtracted() { */ private void readCrate(InputStream stream) throws IOException { File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.exists()) { - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - } else { - FileUtils.forceMkdir(folder); - } + FileSystemUtil.mkdirOrDeleteContent(folder); LocalFileHeader localFileHeader; int readLen; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java index 0a1f9e0f..a7e0ce2e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java @@ -1,6 +1,11 @@ package edu.kit.datamanager.ro_crate.util; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; import java.util.Collection; +import java.util.Objects; import java.util.regex.Matcher; public class FileSystemUtil { @@ -45,4 +50,23 @@ public static String ensureTrailingSlash(String path) { } return path; } + + /** + * Creates a directory or deletes its content if it already exists. + * + * @param folder the folder to create or delete content from + * @throws IOException if an I/O error occurs + */ + public static void mkdirOrDeleteContent(File folder) throws IOException { + boolean isNonEmptyDir = folder.exists() + && folder.isDirectory() + && Objects.requireNonNull(folder.listFiles()).length > 0; + boolean isFile = folder.exists() + && !folder.isDirectory(); + + if (isNonEmptyDir || isFile) { + FileUtils.forceDelete(folder); + } + FileUtils.forceMkdir(folder); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java b/src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java new file mode 100644 index 00000000..fac64527 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java @@ -0,0 +1,31 @@ +package edu.kit.datamanager.ro_crate.util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.*; + +class FileSystemUtilTest { + + @ValueSource(strings = { + "test", + "test/", + "test/test", + "test/test/", + "test/test/test", + "test/test/test/" + }) + @ParameterizedTest + void ensureTrailingSlash(String value) { + String result = FileSystemUtil.ensureTrailingSlash(value); + assertTrue(result.endsWith("/"), "The result should end with a trailing slash."); + } + + @SuppressWarnings("ConstantValue") + @Test + void ensureTrailingSlashNull() { + String result = FileSystemUtil.ensureTrailingSlash(null); + assertNull(result, "The result should be null."); + } +} \ No newline at end of file From 1e2da1171b275f9c0521f7df3a795f0b6fc486c2 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:13:31 +0200 Subject: [PATCH 66/79] refactor: simplify WriteZipStrategy by delegating save operations to WriteZipStreamStrategy --- .../ro_crate/writer/WriteZipStrategy.java | 118 +----------------- 1 file changed, 6 insertions(+), 112 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java index 04812054..003f7973 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java @@ -1,27 +1,13 @@ package edu.kit.datamanager.ro_crate.writer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; -import edu.kit.datamanager.ro_crate.preview.CratePreview; -import edu.kit.datamanager.ro_crate.util.FileSystemUtil; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.model.ZipParameters; -import org.apache.commons.io.FileUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; +import java.io.OutputStream; /** * Implementation of the writing strategy to provide a way of writing crates to @@ -32,108 +18,16 @@ public class WriteZipStrategy implements ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(WriteZipStrategy.class); - public static final String TMP_DIR = "./.tmp/ro-crate-java/writer-zip-strategy/"; - - /** - * Defines if the zip file will directly contain the crate, - * or if it will contain a subdirectory with the crate. - */ - protected boolean createRootSubdir = false; + protected ElnFormatWriter delegate = new WriteZipStreamStrategy(); @Override public ElnFormatWriter usingElnStyle() { - this.createRootSubdir = true; + this.delegate = this.delegate.withRootSubdirectory(); return this; } @Override public void save(Crate crate, String destination) throws IOException { - String innerFolderName = ""; - if (this.createRootSubdir) { - innerFolderName = FileSystemUtil.filterExtensionsFromFileName( - Path.of(destination).getFileName().toString(), - Set.of("ELN", "ZIP")); - innerFolderName = FileSystemUtil.ensureTrailingSlash(innerFolderName); - } - try (ZipFile zipFile = new ZipFile(destination)) { - saveMetadataJson(crate, zipFile, innerFolderName); - saveDataEntities(crate, zipFile, innerFolderName); - savePreview(crate, zipFile, innerFolderName); - } - } - - private void saveDataEntities(Crate crate, ZipFile zipFile, String prefix) throws IOException { - for (DataEntity dataEntity : crate.getAllDataEntities()) { - this.saveToZip(dataEntity, zipFile, prefix); - } - } - - private void saveMetadataJson(Crate crate, ZipFile zipFile, String prefix) throws IOException { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(prefix + "ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - } - } - - private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOException { - Optional preview = Optional.ofNullable(crate.getPreview()); - if (preview.isEmpty()) { - return; - } - final String ID = UUID.randomUUID().toString(); - File tmpPreviewFolder = Path.of(TMP_DIR) - .resolve(ID) - .toFile(); - FileUtils.forceMkdir(tmpPreviewFolder); - FileUtils.forceDeleteOnExit(tmpPreviewFolder); - - preview.get().generate(crate, tmpPreviewFolder); - String[] paths = tmpPreviewFolder.list(); - if (paths == null) { - throw new IOException("No preview files found in temporary folder. Preview generation failed."); - } - for (String path : paths) { - File file = tmpPreviewFolder.toPath().resolve(path).toFile(); - if (file.isDirectory()) { - ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(prefix + path); - parameters.setIncludeRootFolder(false); - zipFile.addFolder(file, parameters); - } else { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(prefix + path); - zipFile.addFile(file, zipParameters); - } - } - try { - FileUtils.forceDelete(tmpPreviewFolder); - } catch (IOException e) { - logger.error("Could not delete temporary preview folder: {}", tmpPreviewFolder); - } - } - - private void saveToZip(DataEntity entity, ZipFile zipFile, String prefix) throws IOException { - if (entity == null || entity.getPath() == null) { - return; - } - - boolean isDirectory = entity.getPath().toFile().isDirectory(); - if (isDirectory) { - ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(prefix + entity.getId()); - parameters.setIncludeRootFolder(false); - zipFile.addFolder(entity.getPath().toFile(), parameters); - } else { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(prefix + entity.getId()); - zipFile.addFile(entity.getPath().toFile(), zipParameters); - } + this.delegate.save(crate, new FileOutputStream(destination)); } } From 9cf26b157dbd72f79830c5556189cae23fb84df7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:25:13 +0200 Subject: [PATCH 67/79] docs: add IOException documentation to readCrate method and clarify CrateWriter type parameter --- .../java/edu/kit/datamanager/ro_crate/reader/CrateReader.java | 2 ++ .../java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 7dcadbdc..5132e44b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -83,6 +83,8 @@ public CrateReader(GenericReaderStrategy strategy) { * * @param location the location of the ro-crate to be read * @return the read RO-crate + * + * @throws IOException if the crate cannot be read */ public RoCrate readCrate(T location) throws IOException { // get the ro-crate-metadata.json diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java index 970803ec..caba67f9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -9,6 +9,8 @@ /** * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) + * + * @param the type which determines the destination of the result */ public class CrateWriter { From 06b2146eab9698794ec1263fb104449c1a72605a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:40:38 +0200 Subject: [PATCH 68/79] test: add not-null-assertions for crate and person entity --- .../ro_crate/crate/realexamples/RealTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java index ae206b7c..19ba2a83 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java @@ -12,7 +12,6 @@ import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.externalproviders.personprovider.OrcidProvider; -import edu.kit.datamanager.ro_crate.reader.CrateReader; import edu.kit.datamanager.ro_crate.reader.Readers; import org.apache.commons.io.FileUtils; @@ -23,17 +22,17 @@ import java.nio.charset.Charset; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -class RealTest { - - @SuppressWarnings("java:S2699") // disable warning about missing assertions +class RealTest +{ @Test void testWithIDRCProject(@TempDir Path temp) throws IOException { - - CrateReader reader = Readers.newFolderReader(); final String locationMetadataFile = "/crates/other/idrc_project/ro-crate-metadata.json"; - Crate crate = reader.readCrate(RealTest.class.getResource("/crates/other/idrc_project").getPath()); + Crate crate = Readers.newFolderReader() + .readCrate(RealTest.class.getResource("/crates/other/idrc_project").getPath()); + assertNotNull(crate); HelpFunctions.compareCrateJsonToFileInResources(crate, locationMetadataFile); Path newFile = temp.resolve("new_file.txt"); @@ -47,6 +46,7 @@ void testWithIDRCProject(@TempDir Path temp) throws IOException { .build()); PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-9842-9718"); + assertNotNull(person); crate.addContextualEntity(person); // problem From 01058be458927bd6c572ef985aff8379f322ee66 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 18:16:09 +0200 Subject: [PATCH 69/79] refactor: improve null safety in mkdirOrDeleteContent by using local variable for file listing --- .../edu/kit/datamanager/ro_crate/util/FileSystemUtil.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java index a7e0ce2e..15c52024 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java @@ -5,7 +5,6 @@ import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.Objects; import java.util.regex.Matcher; public class FileSystemUtil { @@ -58,9 +57,11 @@ public static String ensureTrailingSlash(String path) { * @throws IOException if an I/O error occurs */ public static void mkdirOrDeleteContent(File folder) throws IOException { + File[] files = folder.listFiles(); boolean isNonEmptyDir = folder.exists() && folder.isDirectory() - && Objects.requireNonNull(folder.listFiles()).length > 0; + && files != null + && files.length > 0; boolean isFile = folder.exists() && !folder.isDirectory(); From 7263e78a788d7baa079964ca79b419d1a23c61bb Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 18:29:47 +0200 Subject: [PATCH 70/79] docs: update filterExtensionsFromFileName method documentation to specify null parameter constraints --- .../edu/kit/datamanager/ro_crate/util/FileSystemUtil.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java index 15c52024..e4d75442 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java @@ -20,9 +20,10 @@ private FileSystemUtil() { * Example: * filterExtensionsFromFileName("test.eln", Set.of("ELN")) -> "test" * - * @param filename the file name to filter - * @param extensionsToRemove the extensions to remove + * @param filename the file name to filter (must not be null) + * @param extensionsToRemove the extensions to remove (must not be null) * @return the filtered file name + * @throws NullPointerException if any parameter is null */ public static String filterExtensionsFromFileName(String filename, Collection extensionsToRemove) { String dot = Matcher.quoteReplacement("."); From fea8452ed7fedfa5d39957f1939c1726977256e1 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 18:34:56 +0200 Subject: [PATCH 71/79] refactor: use try-with-resources for FileOutputStream in saveCrate methods --- .../ro_crate/writer/ZipStreamWriterTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java index 3f022cf8..83e311b1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java @@ -23,13 +23,17 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { - new CrateWriter<>(new WriteZipStreamStrategy().usingElnStyle()) - .save(crate, new FileOutputStream(target.toFile())); + try (FileOutputStream stream = new FileOutputStream(target.toFile())) { + new CrateWriter<>(new WriteZipStreamStrategy().usingElnStyle()) + .save(crate, stream); + } } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { - new CrateWriter<>(new WriteZipStreamStrategy().withRootSubdirectory()) - .save(crate, new FileOutputStream(target.toFile())); + try (FileOutputStream stream = new FileOutputStream(target.toFile())) { + new CrateWriter<>(new WriteZipStreamStrategy().withRootSubdirectory()) + .save(crate, stream); + } } } From 2de9b63e6fa29ba004cd9a152fafd477498895ca Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 18:38:11 +0200 Subject: [PATCH 72/79] fix: log error when failing to delete temporary files in CratePreview --- .../edu/kit/datamanager/ro_crate/preview/CratePreview.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index a69b47aa..2459f8b5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -10,6 +10,7 @@ import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; +import org.slf4j.LoggerFactory; /** * Interface for the ROCrate preview. This manages the human-readable @@ -54,6 +55,8 @@ default void generate(Crate crate, File targetDir) throws IOException { } } catch (IOException e) { // Silently ignore deletion errors + LoggerFactory.getLogger(CratePreview.class) + .error("Failed to delete temporary file {}", path, e); } }); } From a6ad7d616453615b9c2d38c87d18cce88dfffa81 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 18:41:00 +0200 Subject: [PATCH 73/79] fix: ensure extracted files are within the target directory using real path validation --- .../kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index afaa217e..3cc41086 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -120,7 +120,8 @@ private void readCrate(InputStream stream) throws IOException { while ((localFileHeader = zipInputStream.getNextEntry()) != null) { String fileName = localFileHeader.getFileName(); File extractedFile = new File(folder, fileName).getCanonicalFile(); - if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { + Path targetRoot = folder.toPath().toRealPath(); + if (!extractedFile.toPath().startsWith(targetRoot)) { throw new IOException("Entry is outside of target directory: " + fileName); } if (localFileHeader.isDirectory()) { From 3db49459a8ae0d53d9f220be97f0e23268dbc5c8 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 18:44:24 +0200 Subject: [PATCH 74/79] refactor: update path handling in addFolderToZipStream to use File object --- .../kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index 8c54cc95..33572db2 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -157,7 +157,7 @@ private void saveToStream(DataEntity entity, ZipOutputStream zipStream, String p if (isDirectory) { ZipStreamUtil.addFolderToZipStream( zipStream, - entity.getPath().toAbsolutePath().toString(), + entity.getPath().toFile(), prefix + entity.getId()); } else { ZipStreamUtil.addFileToZipStream( From d3336ff0be60741882ad70553f1865dfafd180cb Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 19:16:17 +0200 Subject: [PATCH 75/79] refactor: use try-with-resources for InputStream and ZipOutputStream in PreviewTest --- .../ro_crate/preview/PreviewTest.java | 139 ++++++++++-------- 1 file changed, 76 insertions(+), 63 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java index 0c1e67b0..788e0121 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java @@ -1,6 +1,5 @@ package edu.kit.datamanager.ro_crate.preview; -import gg.jte.output.FileOutput; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; @@ -86,12 +85,12 @@ void staticPreviewSaveToZipStream(@TempDir Path dir) throws IOException { FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); StaticPreview preview = new StaticPreview(file1.toFile(), file2.toFile()); - ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(dir.resolve("destination.zip").toFile())); - preview.saveAllToStream( - null, // static preview does not need metadata - stream); - stream.flush(); - stream.close(); + try (ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(dir.resolve("destination.zip").toFile()))) { + preview.saveAllToStream( + null, // static preview does not need metadata + stream); + stream.flush(); + } try (ZipFile zf = new ZipFile(dir.resolve("destination.zip").toFile())) { zf.extractAll(dir.resolve("extracted").toAbsolutePath().toString()); @@ -115,11 +114,13 @@ void staticPreviewSaveToZipStream(@TempDir Path dir) throws IOException { void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); // this crate will not have a json file FileUtils.forceMkdir(crate.toFile()); - FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + try (InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json")) { + Assertions.assertNotNull(crateJson); + FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + } automaticPreview.saveAllToFolder(crate.toFile()); // there should be a html file generated @@ -129,14 +130,14 @@ void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { @Test void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); - zipFile.addStream(crateJson, zipParameters); - crateJson.close(); + try (InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json")) { + zipFile.addStream(crateJson, zipParameters); + } automaticPreview.saveAllToZip(zipFile); @@ -159,82 +160,88 @@ void testAutomaticPreviewZipStream(@TempDir Path dir) throws IOException { Path crate = dir.resolve("crate"); File zipFile = dir.resolve("test.zip").toFile(); - { - ZipFile zip = new ZipFile(zipFile); + try ( + ZipFile zip = new ZipFile(zipFile); + InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath) + ) { ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); - InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath); zip.addStream(crateJson, zipParameters); - crateJson.close(); } - String metadata = new String( - PreviewTest.class.getResourceAsStream(metadataPath) - .readAllBytes()); - ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile)); - preview.saveAllToStream(metadata, stream); - stream.flush(); - stream.close(); + + String metadata; + try (InputStream metadataStream = PreviewTest.class.getResourceAsStream(metadataPath)) { + Assertions.assertNotNull(metadataStream); + metadata = new String(metadataStream.readAllBytes()); + } + + try (ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile))) { + preview.saveAllToStream(metadata, stream); + stream.flush(); + } try { - // this should trow an exception but not stop the execution + // this should throw an exception but not stop the execution ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); preview.saveAllToZip(randomZipFile); Assertions.fail("Expected IOException when providing invalid ZIP file for preview."); } catch (IOException ex) { //ok } - new ZipFile(zipFile).extractAll(crate.toString()); + + try (ZipFile zipReader = new ZipFile(zipFile)) { + zipReader.extractAll(crate.toString()); + } assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } @Test void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { CustomPreview customPreview = new CustomPreview(); - - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); - // this crate will not have a json file Path fakeCrate = dir.resolve("fakeCrate"); FileUtils.forceMkdir(crate.toFile()); - FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + + try (InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json")) { + Assertions.assertNotNull(crateJson); + FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + } customPreview.saveAllToFolder(crate.toFile()); - try { - // this should trow an exception but not stop the execution + try { + // this should throw an exception but not stop the execution customPreview.saveAllToFolder(fakeCrate.toFile()); Assertions.fail("Expected IOException when providing invalid ZIP file for preview."); } catch (IOException ex) { //ok } - // there should be a html file generated assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } @Test void testCustomPreviewZip(@TempDir Path tmp) throws IOException { CustomPreview customPreview = new CustomPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = tmp.resolve("crate"); ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ZipFile zipFile = new ZipFile(tmp.resolve("test.zip").toFile()); - zipFile.addStream(crateJson, zipParameters); - crateJson.close(); - - customPreview.saveAllToZip(zipFile); - - try { - // this should trow an exception but not stop the execution - ZipFile randomZipFile = new ZipFile(tmp.resolve("dddd.zip").toFile()); - customPreview.saveAllToZip(randomZipFile); - Assertions.fail("Expected IOException when providing invalid input to preview."); - } catch (IOException ex) { - //ok + try (ZipFile zipFile = new ZipFile(tmp.resolve("test.zip").toFile()); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json")) { + zipFile.addStream(crateJson, zipParameters); + customPreview.saveAllToZip(zipFile); + + try { + // this should throw an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(tmp.resolve("dddd.zip").toFile()); + customPreview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid input to preview."); + } catch (IOException ex) { + //ok + } + zipFile.extractAll(crate.toString()); } - zipFile.extractAll(crate.toString()); assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } @@ -243,33 +250,39 @@ void testCustomPreviewZipStream(@TempDir Path tmp) throws IOException { CustomPreview preview = new CustomPreview(); String metadataPath = "/crates/other/idrc_project/ro-crate-metadata.json"; Path crate = tmp.resolve("crate"); - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - File zipFile = tmp.resolve("test.zip").toFile(); - { - ZipFile zip = new ZipFile(zipFile); - InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath); + + try (ZipFile zip = new ZipFile(zipFile); + InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath)) { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); zip.addStream(crateJson, zipParameters); - crateJson.close(); } - String metadata = new String( - PreviewTest.class.getResourceAsStream(metadataPath) - .readAllBytes()); - ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile)); - preview.saveAllToStream(metadata, stream); - stream.flush(); - stream.close(); + + String metadata; + try (InputStream metadataStream = PreviewTest.class.getResourceAsStream(metadataPath)) { + Assertions.assertNotNull(metadataStream); + metadata = new String(metadataStream.readAllBytes()); + } + + try (ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile))) { + preview.saveAllToStream(metadata, stream); + stream.flush(); + } try { - // this should trow an exception but not stop the execution + // this should throw an exception but not stop the execution ZipFile randomZipFile = new ZipFile(tmp.resolve("dddd.zip").toFile()); preview.saveAllToZip(randomZipFile); Assertions.fail("Expected IOException when providing invalid input to preview."); } catch (IOException ex) { //ok } - new ZipFile(zipFile).extractAll(crate.toString()); + + try (ZipFile zipReader = new ZipFile(zipFile)) { + zipReader.extractAll(crate.toString()); + } assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } } + From 289ffeea7c7ae1a5d3361f8fb57aee4b5e362b95 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 21 May 2025 14:13:01 +0200 Subject: [PATCH 76/79] fix: increase timeout for downloading ELN file in ElnFileFormatTest Let's give the CI a bit more time. --- .../edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java index bc10848c..dbbb4e05 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -57,7 +57,7 @@ default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOExcept // Download the ELN file URL url = URI.create(urlStr).toURL(); Path elnFile = tmp.resolve("downloaded.eln"); - FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); + FileUtils.copyURLToFile(url, elnFile.toFile(), 20000, 20000); assertTrue(elnFile.toFile().exists()); if (!isInBlacklist(urlStr)) { From d43a8ddcbe9bc063599497a0b6bdbfc738328617 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 21 May 2025 14:27:50 +0200 Subject: [PATCH 77/79] refactor: use try-with-resources for FileInputStream and FileOutputStream in ZipStreamReaderTest --- .../ro_crate/reader/ZipStreamReaderTest.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index 52feda9d..11b82d12 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -29,13 +29,22 @@ public boolean isInBlacklist(String input) { @Override public void saveCrate(Crate crate, Path target) throws IOException { - Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); - assertTrue(target.toFile().isFile()); + final File target_file = target.toFile(); + try ( + FileOutputStream fos = new FileOutputStream(target_file) + ) { + Writers.newZipStreamWriter().save(crate, fos); + } + assertTrue(target_file.isFile()); } @Override public Crate readCrate(Path source) throws IOException { - return Readers.newZipStreamReader().readCrate(new FileInputStream(source.toFile())); + try ( + FileInputStream fis = new FileInputStream(source.toFile()) + ) { + return Readers.newZipStreamReader().readCrate(fis); + } } @Override @@ -53,9 +62,13 @@ public ReadZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean @Override public Crate readCrate(ReadZipStreamStrategy strategy, Path source) throws IOException { - Crate importedCrate = new CrateReader<>(strategy) - .readCrate(new FileInputStream(source.toFile())); - assertTrue(strategy.isExtracted()); - return importedCrate; + try ( + FileInputStream fis = new FileInputStream(source.toFile()) + ) { + Crate importedCrate = new CrateReader<>(strategy).readCrate(fis); + assertNotNull(importedCrate); + assertTrue(strategy.isExtracted()); + return importedCrate; + } } } From 684734b8fe1156dddd272ef07dd23ddfd4c9997d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 21 May 2025 15:54:58 +0200 Subject: [PATCH 78/79] refactor: ensure inconsistencies are being fixed when adding an id property This never happened in the tests, but may be useful when reading inconsistent crates. --- .../kit/datamanager/ro_crate/entities/AbstractEntity.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 34889d21..3391de8f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -239,12 +239,13 @@ private static boolean addProperty(ObjectNode whereToAdd, String key, JsonNode v * @param id the "id" of the property. */ public void addIdProperty(String name, String id) { + if (id == null || id.isBlank()) { return; } mergeIdIntoValue(id, this.properties.get(name)) .ifPresent(newValue -> { - this.linkedTo.add(id); this.properties.set(name, newValue); - this.notifyObservers(); }); + this.linkedTo.add(id); + this.notifyObservers(); } /** From ee9bdb1219ae23dc78a17911e2a2880c8dcb887f Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 21 May 2025 16:55:10 +0200 Subject: [PATCH 79/79] refactor: change mergeIdIntoValue method visibility to protected and add unit tests --- .../ro_crate/entities/AbstractEntity.java | 2 +- .../ro_crate/entities/AbstractEntityTest.java | 111 ++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/entities/AbstractEntityTest.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 3391de8f..f0c75763 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -264,7 +264,7 @@ public void addIdProperty(String name, String id) { * @return The updated value of the property. * Empty if value does not change! */ - private static Optional mergeIdIntoValue(String id, JsonNode currentValue) { + protected static Optional mergeIdIntoValue(String id, JsonNode currentValue) { if (id == null || id.isBlank()) { return Optional.empty(); } ObjectMapper jsonBuilder = MyObjectMapper.getMapper(); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/entities/AbstractEntityTest.java b/src/test/java/edu/kit/datamanager/ro_crate/entities/AbstractEntityTest.java new file mode 100644 index 00000000..678642ba --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/entities/AbstractEntityTest.java @@ -0,0 +1,111 @@ +package edu.kit.datamanager.ro_crate.entities; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; + +class AbstractEntityTest { + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" ", "\t", "\n"}) + void mergeIdIntoValue_withInvalidId_returnsEmpty(String invalidId) { + Optional result = AbstractEntity.mergeIdIntoValue(invalidId, null); + assertTrue(result.isEmpty(), "Should return empty Optional for invalid ID"); + + Optional result2 = AbstractEntity.mergeIdIntoValue( + invalidId, + MyObjectMapper.getMapper().createObjectNode() + ); + assertTrue(result2.isEmpty(), "Should return empty Optional for invalid ID"); + } + + @Test + void mergeIdIntoValue_withNullCurrentValue_returnsIdObject() { + String id = "test-id"; + Optional result = AbstractEntity.mergeIdIntoValue(id, null); + + // Should return a value for valid ID + JsonNode node = result.orElseThrow(); + assertTrue(node.isObject(), "Should return an object"); + assertEquals(id, node.get("@id").asText(), "Should contain the ID"); + } + + @Test + void mergeIdIntoValue_withExistingIdAsString_returnsEmpty() { + String id = "test-id"; + JsonNode currentValue = MyObjectMapper.getMapper().valueToTree(id); + + Optional result = AbstractEntity.mergeIdIntoValue(id, currentValue); + assertTrue(result.isEmpty(), "Should return empty when ID already exists as string"); + } + + @Test + void mergeIdIntoValue_withExistingIdObject_returnsEmpty() { + String id = "test-id"; + ObjectNode currentValue = MyObjectMapper.getMapper().createObjectNode(); + currentValue.put("@id", id); + + Optional result = AbstractEntity.mergeIdIntoValue(id, currentValue); + assertTrue(result.isEmpty(), "Should return empty when ID already exists as object"); + } + + @Test + void mergeIdIntoValue_withNonArrayValue_createsArray() { + String id = "new-id"; + String existingId = "existing-id"; + ObjectNode existingValue = MyObjectMapper.getMapper().createObjectNode(); + existingValue.put("@id", existingId); + + Optional result = AbstractEntity.mergeIdIntoValue(id, existingValue); + + assertTrue(result.isPresent(), "Should return a value"); + JsonNode node = result.get(); + assertTrue(node.isArray(), "Should be converted to array"); + assertEquals(2, node.size(), "Should contain both values"); + assertEquals(existingId, node.get(0).get("@id").asText(), "Should contain existing ID"); + assertEquals(id, node.get(1).get("@id").asText(), "Should contain new ID"); + } + + @Test + void mergeIdIntoValue_withExistingArray_addsToArray() { + String id = "new-id"; + String existingId = "existing-id"; + + ArrayNode currentValue = MyObjectMapper.getMapper().createArrayNode(); + ObjectNode existingIdObj = MyObjectMapper.getMapper().createObjectNode(); + existingIdObj.put("@id", existingId); + currentValue.add(existingIdObj); + + Optional result = AbstractEntity.mergeIdIntoValue(id, currentValue); + + assertTrue(result.isPresent(), "Should return a value"); + JsonNode node = result.get(); + assertTrue(node.isArray(), "Should remain an array"); + assertEquals(2, node.size(), "Should contain both values"); + assertEquals(existingId, node.get(0).get("@id").asText(), "Should contain existing ID"); + assertEquals(id, node.get(1).get("@id").asText(), "Should contain new ID"); + } + + @Test + void mergeIdIntoValue_withExistingArrayContainingId_returnsEmpty() { + String id = "test-id"; + ArrayNode currentValue = MyObjectMapper.getMapper().createArrayNode(); + ObjectNode existingIdObj = MyObjectMapper.getMapper().createObjectNode(); + existingIdObj.put("@id", id); + currentValue.add(existingIdObj); + + Optional result = AbstractEntity.mergeIdIntoValue(id, currentValue); + assertTrue(result.isEmpty(), "Should return empty when ID already exists in array"); + } +} +