From cdecbed8f6db4bbca42705209a73ffdb3bcd22da Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 12:40:54 +0200 Subject: [PATCH 01/46] test: add parameterized test for reading ELN crates from URLs --- .../ro_crate/reader/ZipReaderTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 3611c834..25babd81 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -2,14 +2,52 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.writer.Writers; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.net.URL; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; class ZipReaderTest extends CrateReaderTest { + /** + * ELN Crates are zip files not fully compatible with the Ro-Crate standard + * in the sense that they must contain a single subfolder in the zip file + * which then contain a crate as specified by the Ro-Crate standard. + *

+ * Here we test if we can read them using out ZipReader. + * + * @see + */ + @ParameterizedTest + @ValueSource(strings = { + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/AI4Green/Export%20workbook-2024-08-27-export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/OpenSemanticLab/MinimalExample.osl.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/PASTA/PASTA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/RSpace/RSpace-2023-12-08-14-44-xml-SELECTION-c0bEtpHcnNe-HA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/SampleDB/sampledb_export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/elabftw/export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" + }) + void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { + // Download the ELN file + URL url = new URL(urlStr); + Path elnFile = tmp.resolve("downloaded.eln"); + FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); + assertTrue(elnFile.toFile().exists()); + + // Read the crate from the downloaded file + Crate read = this.readCrate(elnFile); + assertNotNull(read); + assertFalse(read.getAllDataEntities().isEmpty()); + } + @Override protected void saveCrate(Crate crate, Path target) { Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); From c7e0afceebcec71d133b65fa7c1f8fc8ac802b0b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 13:41:44 +0200 Subject: [PATCH 02/46] refactor: enable readers to implement subsets of tests if they make sense for them. --- ...eReaderTest.java => CommonReaderTest.java} | 60 +++++-------------- .../ro_crate/reader/ElnFileFormatTest.java | 53 ++++++++++++++++ .../ro_crate/reader/FolderReaderTest.java | 12 ++-- .../reader/TestableReaderStrategy.java | 50 ++++++++++++++++ .../ro_crate/reader/ZipReaderTest.java | 52 +++------------- .../ro_crate/reader/ZipStreamReaderTest.java | 13 ++-- 6 files changed, 139 insertions(+), 101 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/reader/{CrateReaderTest.java => CommonReaderTest.java} (78%) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java similarity index 78% rename from src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java index bcf8948d..6cdc6fa1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java @@ -27,9 +27,13 @@ * This parameter is only required to satisfy the generic reader strategy. * @param the type of the reader strategy */ -abstract class CrateReaderTest> { - - protected static RoCrate.RoCrateBuilder newBaseCrate() { +interface CommonReaderTest< + SOURCE_T, + READER_STRATEGY extends GenericReaderStrategy + > + extends TestableReaderStrategy +{ + static RoCrate.RoCrateBuilder newBaseCrate() { return new RoCrate.RoCrateBuilder( "minimal", "minimal RO_crate", @@ -38,7 +42,7 @@ protected static RoCrate.RoCrateBuilder newBaseCrate() { ); } - protected static FileEntity newDataEntity(Path filePath) throws IllegalArgumentException { + static FileEntity newDataEntity(Path filePath) throws IllegalArgumentException { return new FileEntity.FileEntityBuilder() .setLocationWithExceptions(filePath) .setId(filePath.toFile().getName()) @@ -48,44 +52,8 @@ protected static FileEntity newDataEntity(Path filePath) throws IllegalArgumentE .build(); } - /** - * Saves the crate with the writer fitting to the reader of {@link #readCrate(Path)}. - * - * @param crate the crate to save - * @param target the target path to the save location - * @throws IOException if an error occurs while saving the crate - */ - abstract protected void saveCrate(Crate crate, Path target) throws IOException; - - /** - * Reads the crate with the reader fitting to the writer of {@link #saveCrate(Crate, Path)}. - * @param source the source path to the crate - * @return the read crate - * @throws IOException if an error occurs while reading the crate - */ - abstract protected Crate readCrate(Path source) throws IOException; - - /** - * Creates a new reader strategy with a non-default temporary directory (if supported, default otherwise). - * - * @param tmpDirectory the temporary directory to use - * @param useUuidSubfolder whether to create a UUID subfolder under the temporary directory - * @return a new reader strategy - */ - abstract protected READER_STRATEGY newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder); - - /** - * Reads the crate using the provided reader strategy. - * - * @param strategy the reader strategy to use - * @param source the source path to the crate - * @return the read crate - * @throws IOException if an error occurs while reading the crate - */ - abstract protected Crate readCrate(READER_STRATEGY strategy, Path source) throws IOException; - @Test - void testReadingBasicCrate(@TempDir Path temp) throws IOException { + default void testReadingBasicCrate(@TempDir Path temp) throws IOException { RoCrate roCrate = newBaseCrate().build(); Path zipPath = temp.resolve("result.zip"); @@ -95,7 +63,7 @@ void testReadingBasicCrate(@TempDir Path temp) throws IOException { } @Test - void testWithFile(@TempDir Path temp) throws IOException { + default void testWithFile(@TempDir Path temp) throws IOException { Path csvPath = temp.resolve("survey-responses-2019.csv"); FileUtils.touch(csvPath.toFile()); FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); @@ -113,7 +81,7 @@ void testWithFile(@TempDir Path temp) throws IOException { } @Test - void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { + default void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { // This URL will be encoded because of whitespaces Path csvPath = temp.resolve("survey responses 2019.csv"); FileUtils.touch(csvPath.toFile()); @@ -140,7 +108,7 @@ void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { } @Test - void TestWithFileWithLocation(@TempDir Path temp) throws IOException { + default void TestWithFileWithLocation(@TempDir Path temp) throws IOException { Path csvPath = temp.resolve("survey-responses-2019.csv"); FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); RoCrate rawCrate = newBaseCrate() @@ -168,7 +136,7 @@ void TestWithFileWithLocation(@TempDir Path temp) throws IOException { } @Test - void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { + default void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { Path csvPath = temp.resolve("file.csv"); FileUtils.writeStringToFile(csvPath.toFile(), "fakecsv.1", Charset.defaultCharset()); RoCrate rawCrate = newBaseCrate() @@ -206,7 +174,7 @@ void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { } @Test - void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { + default void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { RoCrate rawCrate = newBaseCrate().build(); // Write to zip file diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java new file mode 100644 index 00000000..5aa01b80 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -0,0 +1,53 @@ +package edu.kit.datamanager.ro_crate.reader; + +import edu.kit.datamanager.ro_crate.Crate; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +public interface ElnFileFormatTest< + SOURCE_T, + READER_STRATEGY extends GenericReaderStrategy + > + extends TestableReaderStrategy +{ + /** + * ELN Crates are zip files not fully compatible with the Ro-Crate standard + * in the sense that they must contain a single subfolder in the zip file + * which then contain a crate as specified by the Ro-Crate standard. + *

+ * Here we test if we can read them using out ZipReader. + * + * @see + */ + @ParameterizedTest + @ValueSource(strings = { + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/AI4Green/Export%20workbook-2024-08-27-export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/OpenSemanticLab/MinimalExample.osl.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/PASTA/PASTA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/RSpace/RSpace-2023-12-08-14-44-xml-SELECTION-c0bEtpHcnNe-HA.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/SampleDB/sampledb_export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/elabftw/export.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" + }) + default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { + // Download the ELN file + URL url = new URL(urlStr); + Path elnFile = tmp.resolve("downloaded.eln"); + FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); + assertTrue(elnFile.toFile().exists()); + + // Read the crate from the downloaded file + Crate read = this.readCrate(elnFile); + assertNotNull(read); + assertFalse(read.getAllDataEntities().isEmpty()); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index 21850edd..a7143862 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -17,21 +17,21 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderReaderTest extends CrateReaderTest { - +class FolderReaderTest implements CommonReaderTest +{ @Override - protected void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) { Writers.newFolderWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isDirectory()); } @Override - protected Crate readCrate(Path source) throws IOException { + public Crate readCrate(Path source) throws IOException { return Readers.newFolderReader().readCrate(source.toAbsolutePath().toString()); } @Override - protected FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { // This strategy does not support a non-default temporary directory // and will always use the default one. // It also has no state we could make assertions on. @@ -39,7 +39,7 @@ protected FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean use } @Override - protected Crate readCrate(FolderStrategy strategy, Path source) throws IOException { + public Crate readCrate(FolderStrategy strategy, Path source) throws IOException { return new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java new file mode 100644 index 00000000..c8d4b72a --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/TestableReaderStrategy.java @@ -0,0 +1,50 @@ +package edu.kit.datamanager.ro_crate.reader; + +import edu.kit.datamanager.ro_crate.Crate; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Base Interface for methods required to test all reader strategies. + * + * @param the source type the strategy reads from. + * @param the type of the reader strategy. + */ +interface TestableReaderStrategy> { + /** + * Saves the crate with the writer fitting to the reader of {@link #readCrate(Path)}. + * + * @param crate the crate to save + * @param target the target path to the save location + * @throws IOException if an error occurs while saving the crate + */ + void saveCrate(Crate crate, Path target) throws IOException; + + /** + * Reads the crate with the reader fitting to the writer of {@link #saveCrate(Crate, Path)}. + * @param source the source path to the crate + * @return the read crate + * @throws IOException if an error occurs while reading the crate + */ + Crate readCrate(Path source) throws IOException; + + /** + * Creates a new reader strategy with a non-default temporary directory (if supported, default otherwise). + * + * @param tmpDirectory the temporary directory to use + * @param useUuidSubfolder whether to create a UUID subfolder under the temporary directory + * @return a new reader strategy + */ + READER_STRATEGY newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder); + + /** + * Reads the crate using the provided reader strategy. + * + * @param strategy the reader strategy to use + * @param source the source path to the crate + * @return the read crate + * @throws IOException if an error occurs while reading the crate + */ + Crate readCrate(READER_STRATEGY strategy, Path source) throws IOException; +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 25babd81..a4616f8e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -2,65 +2,29 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.writer.Writers; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; -import java.net.URL; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -class ZipReaderTest extends CrateReaderTest { - - /** - * ELN Crates are zip files not fully compatible with the Ro-Crate standard - * in the sense that they must contain a single subfolder in the zip file - * which then contain a crate as specified by the Ro-Crate standard. - *

- * Here we test if we can read them using out ZipReader. - * - * @see - */ - @ParameterizedTest - @ValueSource(strings = { - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/AI4Green/Export%20workbook-2024-08-27-export.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/OpenSemanticLab/MinimalExample.osl.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/PASTA/PASTA.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/RSpace/RSpace-2023-12-08-14-44-xml-SELECTION-c0bEtpHcnNe-HA.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/SampleDB/sampledb_export.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/elabftw/export.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", - "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" - }) - void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { - // Download the ELN file - URL url = new URL(urlStr); - Path elnFile = tmp.resolve("downloaded.eln"); - FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); - assertTrue(elnFile.toFile().exists()); - - // Read the crate from the downloaded file - Crate read = this.readCrate(elnFile); - assertNotNull(read); - assertFalse(read.getAllDataEntities().isEmpty()); - } - +class ZipReaderTest implements + CommonReaderTest, + ElnFileFormatTest +{ @Override - protected void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) { Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isFile()); } @Override - protected Crate readCrate(Path source) throws IOException { + public Crate readCrate(Path source) throws IOException { return Readers.newZipPathReader().readCrate(source.toAbsolutePath().toString()); } @Override - protected ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { ZipStrategy strategy = new ZipStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { @@ -73,7 +37,7 @@ protected ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUui } @Override - protected Crate readCrate(ZipStrategy strategy, Path source) throws IOException { + public Crate readCrate(ZipStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); assertTrue(strategy.isExtracted()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index 33436c6c..17b37247 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -9,20 +9,23 @@ import static org.junit.jupiter.api.Assertions.*; -class ZipStreamReaderTest extends CrateReaderTest { +class ZipStreamReaderTest implements + CommonReaderTest, + ElnFileFormatTest +{ @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); assertTrue(target.toFile().isFile()); } @Override - protected Crate readCrate(Path source) throws IOException { + public Crate readCrate(Path source) throws IOException { return Readers.newZipStreamReader().readCrate(new FileInputStream(source.toFile())); } @Override - protected ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { ZipStreamStrategy strategy = new ZipStreamStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { @@ -35,7 +38,7 @@ protected ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean } @Override - protected Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { + public Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(new FileInputStream(source.toFile())); assertTrue(strategy.isExtracted()); From 2e0df3510c13e3e42c25201e4d6181a80401a736 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 14:46:49 +0200 Subject: [PATCH 03/46] feat: reading zip files now searches for a crate in subdirectories. Not recursive and limited to 50 folders to avoid load vulnerabilities. --- .../entities/contextual/JsonDescriptor.java | 4 ++-- .../datamanager/ro_crate/reader/ZipStrategy.java | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java index 8bb91294..18acd4c6 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java @@ -13,8 +13,8 @@ public class JsonDescriptor extends ContextualEntity { - private static final String CONFORMS_TO = "conformsTo"; - protected static final String ID = "ro-crate-metadata.json"; + protected static final String CONFORMS_TO = "conformsTo"; + public static final String ID = "ro-crate-metadata.json"; /** * Returns a JsonDescriptor with the conformsTo value set to the latest stable diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java index 0d6381a6..26118841 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; @@ -106,7 +107,17 @@ public ObjectNode readMetadataJson(String location) { } ObjectMapper objectMapper = MyObjectMapper.getMapper(); - File jsonMetadata = temporaryFolder.resolve("ro-crate-metadata.json").toFile(); + File jsonMetadata = this.temporaryFolder.resolve(JsonDescriptor.ID).toFile(); + if (!jsonMetadata.isFile()) { + // Try to find the metadata file in subdirectories + File firstSubdir = FileUtils.listFilesAndDirs(temporaryFolder.toFile(), null, null) + .stream() + .limit(50) + .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); + jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); + } try { return objectMapper.readTree(jsonMetadata).deepCopy(); From ccc0fe841e560b1008dfd6972e53171149aa6719 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:03:20 +0200 Subject: [PATCH 04/46] chore: fix deprecation warning in ElnFileFormatTest and enable better deprecation linter errors --- build.gradle | 5 +++++ .../kit/datamanager/ro_crate/reader/ElnFileFormatTest.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7a9d9647..f7fcd613 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,11 @@ dependencies { implementation("org.freemarker:freemarker:2.3.34") } +// enable -Xlint:deprecation +tasks.withType(JavaCompile).configureEach { + options.compilerArgs << "-Xlint:deprecation" +} + logging.captureStandardOutput LogLevel.INFO def signingTasks = tasks.withType(Sign) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java index 5aa01b80..3e06fc7f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -7,6 +7,7 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.file.Path; @@ -40,7 +41,7 @@ public interface ElnFileFormatTest< }) default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOException { // Download the ELN file - URL url = new URL(urlStr); + URL url = URI.create(urlStr).toURL(); Path elnFile = tmp.resolve("downloaded.eln"); FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); assertTrue(elnFile.toFile().exists()); From 0434a737e4bb8651c2403277e12149b2c2fc5635 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:04:21 +0200 Subject: [PATCH 05/46] fix: improve error reporting for JSON schema validation --- .../ro_crate/entities/validation/JsonSchemaValidation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java index 067f5e2e..18e9624a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java @@ -60,6 +60,7 @@ public boolean validateEntity(JsonNode entity) { Set errors = this.entitySchema.validate(entity); if (errors.size() != 0) { System.err.println("This entity does not comply to the basic RO-Crate entity structure."); + errors.forEach(error -> System.err.println(error.getMessage())); return false; } return true; From 966255fb0788c12242bc390c20a29b884974e124 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:52:16 +0200 Subject: [PATCH 06/46] refactor: replace deprecated setAll method with setAllIfValid and setAllUnsafe in entity builders --- .../ro_crate/entities/AbstractEntity.java | 46 +++++++++++++++++-- .../dataentities/ImportFromZenodo.java | 4 +- .../personprovider/OrcidProvider.java | 2 +- .../ro_crate/reader/CrateReader.java | 8 ++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 8699fd5b..d3afdc6c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -558,12 +558,19 @@ public T addIdFromCollectionOfEntities(String name, Collection e } /** - * This sets everything from a json object to the property. Can be - * useful when the entity is already available somewhere. + * Deprecated. Equivalent to {@link #setAllIfValid(ObjectNode)}. * * @param properties the Json representing all the properties. - * @return the generic builder. + * @return the generic builder, either including all given properties + * * or unchanged. + * + * @deprecated To enforce the user know what this method does, + * we want the user to use one of the more explicitly named + * methods {@link #setAllIfValid(ObjectNode)} or + * {@link #setAllUnsafe(ObjectNode)}. + * @see #setAllIfValid(ObjectNode) */ + @Deprecated(since = "2.1.0", forRemoval = true) public T setAll(ObjectNode properties) { if (AbstractEntity.entityValidation.entityValidation(properties)) { this.properties = properties; @@ -572,6 +579,39 @@ public T setAll(ObjectNode properties) { return self(); } + /** + * This sets everything from a json object to the property, + * if the result is valid. Otherwise, it will do nothing. + * + * @param properties the Json representing all the properties. + * @return the generic builder, either including all given properties + * or unchanged. + */ + public T setAllIfValid(ObjectNode properties) { + if (AbstractEntity.entityValidation.entityValidation(properties)) { + this.properties = properties; + this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties)); + } + return self(); + } + + /** + * This sets everything from a json object to the property. Can be + * useful when the entity is already available somewhere. + *

+ * Errors on validation are printed, but everything will be added. + * + * @param properties the Json representing all the properties. + * @return the generic builder with all properties added. + */ + public T setAllUnsafe(ObjectNode properties) { + // This will currently only print errors. + AbstractEntity.entityValidation.entityValidation(properties); + this.properties = properties; + this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties)); + return self(); + } + public abstract T self(); public abstract AbstractEntity build(); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java index 1be5607b..4a99ee8a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java @@ -112,12 +112,12 @@ private static void addToCrateFromZotero(String url, Crate crate) { for (var entity : graph) { if (entity.get("@id").asText().equals(mainId)) { var dataEntity = new DataEntity.DataEntityBuilder() - .setAll((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addDataEntity(dataEntity); } else { // here we have to think of a way to differentiate between data and contextual entities. var contextualEntity = new ContextualEntity.ContextualEntityBuilder() - .setAll((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addContextualEntity(contextualEntity); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java index 8e079af9..17207993 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java @@ -72,7 +72,7 @@ public static PersonEntity getPerson(String url) { node.set(element.getKey(), element.getValue()); } } - return new PersonEntity.PersonEntityBuilder().setAll(node).build(); + return new PersonEntity.PersonEntityBuilder().setAllUnsafe(node).build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); logger.error(errorMessage); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 5acb575b..ba2c2ae8 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -119,7 +119,7 @@ private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet { @@ -133,7 +133,7 @@ private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet extractHasPartIds(ObjectNode root) { private void setCrateDescriptor(RoCrate crate, JsonNode descriptor) { ContextualEntity descriptorEntity = new ContextualEntity.ContextualEntityBuilder() - .setAll(descriptor.deepCopy()) + .setAllUnsafe(descriptor.deepCopy()) .build(); crate.setJsonDescriptor(descriptorEntity); } From d0c1103f9cd6c5387d8355015b332e77fba7a335 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:52:45 +0200 Subject: [PATCH 07/46] refactor: use try-with-resources for CloseableHttpClient in RoCrateMetadataContext --- .../datamanager/ro_crate/context/RoCrateMetadataContext.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java index 731a4c1a..9eac5a3f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java @@ -174,10 +174,9 @@ public void addToContextFromUrl(String url) { } } if (jsonNode == null) { - CloseableHttpClient httpclient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(url); CloseableHttpResponse response; - try { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { response = httpclient.execute(httpGet); jsonNode = objectMapper.readValue(response.getEntity().getContent(), JsonNode.class); From cdb897c0a3c2d536a0504b9d0bb4c5539666b7ca Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 16:53:30 +0200 Subject: [PATCH 08/46] fix: correct typos and improve formatting in documentation and error messages --- .../ro_crate/context/RoCrateMetadataContext.java | 9 +++++---- .../ro_crate/entities/contextual/JsonDescriptor.java | 2 +- .../edu/kit/datamanager/ro_crate/reader/CrateReader.java | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java index 9eac5a3f..cce6a7e9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/context/RoCrateMetadataContext.java @@ -113,9 +113,10 @@ public boolean checkEntity(AbstractEntity entity) { node.remove("@id"); node.remove("@type"); - Set types = objectMapper.convertValue(entity.getProperties().get("@type"), - new TypeReference<>() { - }); + Set types = objectMapper.convertValue( + entity.getProperties().path("@type"), + new TypeReference<>() {} + ); // check if the items in the array of types are present in the context for (String s : types) { // special cases: @@ -181,7 +182,7 @@ public void addToContextFromUrl(String url) { jsonNode = objectMapper.readValue(response.getEntity().getContent(), JsonNode.class); } catch (IOException e) { - System.err.println(String.format("Cannot get context from url %s", url)); + System.err.printf("Cannot get context from url %s%n", url); return; } if (url.equals(DEFAULT_CONTEXT)) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java index 18acd4c6..88aaf89e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/contextual/JsonDescriptor.java @@ -39,7 +39,7 @@ private JsonDescriptor(ContextualEntityBuilder builder) { /** * Builder for the JsonDescriptor. - * + *

* Defaults to the latest stable crate version and no other conformsTo values. */ public static final class Builder { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index ba2c2ae8..6586f1ed 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -29,7 +29,7 @@ * The constructor takes a strategy to support different ways of importing the * crates. (from zip, folder, etc.). *

- * The reader consideres "hasPart" and "isPartOf" properties and considers all + * The reader considers "hasPart" and "isPartOf" properties and considers all * entities (in-)directly connected to the root entity ("./") as DataEntities. * * @param the type of the location parameter From 160ba9730c0edd3fb86a75c5b4a0bbf77e5aaf57 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 17:54:08 +0200 Subject: [PATCH 09/46] docs: enhance documentation and add tests for setAllIfValid and setAllUnsafe methods --- .../ro_crate/entities/AbstractEntity.java | 8 +- .../contextual/ContextualEntityTest.java | 105 +++++++++++++++++- 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index d3afdc6c..3ec7bb9f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -567,7 +567,7 @@ public T addIdFromCollectionOfEntities(String name, Collection e * @deprecated To enforce the user know what this method does, * we want the user to use one of the more explicitly named * methods {@link #setAllIfValid(ObjectNode)} or - * {@link #setAllUnsafe(ObjectNode)}. + * {@link #setAllIfValid(ObjectNode)}. * @see #setAllIfValid(ObjectNode) */ @Deprecated(since = "2.1.0", forRemoval = true) @@ -582,6 +582,11 @@ public T setAll(ObjectNode properties) { /** * This sets everything from a json object to the property, * if the result is valid. Otherwise, it will do nothing. + *

+ * Valid means here that the json object needs to be flat as specified + * in the RO-Crate specification. In principle, this means that + * primitives and objects referencing an ID are allowed, + * as well as arrays of these. * * @param properties the Json representing all the properties. * @return the generic builder, either including all given properties @@ -600,6 +605,7 @@ public T setAllIfValid(ObjectNode properties) { * useful when the entity is already available somewhere. *

* Errors on validation are printed, but everything will be added. + * For more about validation, see {@link #setAllIfValid(ObjectNode)}. * * @param properties the Json representing all the properties. * @return the generic builder with all properties added. diff --git a/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java b/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java index 575c941e..7bc868f3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/entities/contextual/ContextualEntityTest.java @@ -1,13 +1,16 @@ package edu.kit.datamanager.ro_crate.entities.contextual; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.io.IOException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + /** * @author Nikola Tzotchev on 5.2.2022 г. * @version 1 @@ -45,4 +48,102 @@ void testSerialization() throws IOException { assertTrue(place.getLinkedTo().contains(geo.getId())); HelpFunctions.compareEntityWithFile(place, "/json/entities/contextual/place.json"); } + + @Test + void testAddAllValidCase() throws JsonProcessingException { + ContextualEntity first = new ContextualEntity.ContextualEntityBuilder() + .setId("#b4168a98-8534-4c6d-a568-64a55157b656") + .addType("GeoCoordinates") + .addProperty("latitude", "-33.7152") + .addProperty("longitude", "150.30119") + .addProperty("name", "Latitude: -33.7152 Longitude: 150.30119") + .build(); + + String allProperties = """ + { + "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", + "@type": "GeoCoordinates", + "latitude": "-33.7152", + "longitude": "150.30119", + "name": "Latitude: -33.7152 Longitude: 150.30119" + } + """; + + ObjectNode properties = MyObjectMapper.getMapper() + .readValue(allProperties, ObjectNode.class); + ContextualEntity second = new ContextualEntity.ContextualEntityBuilder() + .setAllIfValid(properties) + .build(); + assertEquals(second.getProperties(), first.getProperties()); + } + + @Test + void testAddAllInvalidCase() throws JsonProcessingException { + ContextualEntity first = new ContextualEntity.ContextualEntityBuilder() + .setId("#b4168a98-8534-4c6d-a568-64a55157b656") + .addType("GeoCoordinates") + .addProperty("latitude", "-33.7152") + .addProperty("longitude", "150.30119") + .addProperty("name", "Latitude: -33.7152 Longitude: 150.30119") + .build(); + + String allProperties = """ + { + "wrong property": {"any": "value"}, + "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", + "@type": "GeoCoordinates", + "latitude": "-33.7152", + "longitude": "150.30119", + "name": "Latitude: -33.7152 Longitude: 150.30119" + } + """; + + ObjectNode properties = MyObjectMapper.getMapper() + .readValue(allProperties, ObjectNode.class); + ContextualEntity second = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .setAllIfValid(properties) + .build(); + assertNotEquals(second.getProperties(), first.getProperties()); + ObjectNode empty = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .build() + .getProperties(); + assertEquals(empty, second.getProperties()); + } + + @Test + void testAddAllUnsafeDoesInvalidCase() throws JsonProcessingException { + ContextualEntity first = new ContextualEntity.ContextualEntityBuilder() + .setId("#b4168a98-8534-4c6d-a568-64a55157b656") + .addType("GeoCoordinates") + .addProperty("latitude", "-33.7152") + .addProperty("longitude", "150.30119") + .addProperty("name", "Latitude: -33.7152 Longitude: 150.30119") + .build(); + + String allProperties = """ + { + "wrong property": {"any": "value"}, + "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", + "@type": "GeoCoordinates", + "latitude": "-33.7152", + "longitude": "150.30119", + "name": "Latitude: -33.7152 Longitude: 150.30119" + } + """; + + ObjectNode properties = MyObjectMapper.getMapper() + .readValue(allProperties, ObjectNode.class); + ContextualEntity second = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .setAllUnsafe(properties) + .build(); + assertNotEquals(second.getProperties(), first.getProperties()); + ObjectNode empty = new ContextualEntity.ContextualEntityBuilder() + .setId("second") + .build() + .getProperties(); + assertNotEquals(empty, second.getProperties()); + } } From 86654f65fbb5dcaf1f6df75ace43524c93331ed4 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 5 May 2025 17:58:09 +0200 Subject: [PATCH 10/46] fix: invalid entities from our providers should be logged and not written We only want to add it anyway when reading, but avoid when reading a crate. --- .../externalproviders/dataentities/ImportFromZenodo.java | 4 ++-- .../externalproviders/personprovider/OrcidProvider.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java index 4a99ee8a..2cc6680a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java @@ -112,12 +112,12 @@ private static void addToCrateFromZotero(String url, Crate crate) { for (var entity : graph) { if (entity.get("@id").asText().equals(mainId)) { var dataEntity = new DataEntity.DataEntityBuilder() - .setAllUnsafe((ObjectNode) entity).build(); + .setAllIfValid((ObjectNode) entity).build(); crate.addDataEntity(dataEntity); } else { // here we have to think of a way to differentiate between data and contextual entities. var contextualEntity = new ContextualEntity.ContextualEntityBuilder() - .setAllUnsafe((ObjectNode) entity).build(); + .setAllIfValid((ObjectNode) entity).build(); crate.addContextualEntity(contextualEntity); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java index 17207993..dd0b802b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java @@ -72,7 +72,7 @@ public static PersonEntity getPerson(String url) { node.set(element.getKey(), element.getValue()); } } - return new PersonEntity.PersonEntityBuilder().setAllUnsafe(node).build(); + return new PersonEntity.PersonEntityBuilder().setAllIfValid(node).build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); logger.error(errorMessage); From 7e4a30f3c6dc4abac4970011c16d5e2a51a7d2bc Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 16:02:36 +0200 Subject: [PATCH 11/46] refactor: optimize file search in ZipStrategy by using directory filter --- .../ro_crate/reader/ZipStrategy.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java index 26118841..74622b2f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -6,6 +6,7 @@ import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; import java.io.File; import java.io.IOException; @@ -110,12 +111,16 @@ public ObjectNode readMetadataJson(String location) { File jsonMetadata = this.temporaryFolder.resolve(JsonDescriptor.ID).toFile(); if (!jsonMetadata.isFile()) { // Try to find the metadata file in subdirectories - File firstSubdir = FileUtils.listFilesAndDirs(temporaryFolder.toFile(), null, null) - .stream() - .limit(50) - .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); + File firstSubdir = FileUtils.listFilesAndDirs( + temporaryFolder.toFile(), + FileFilterUtils.directoryFileFilter(), + null // not recursive + ) + .stream() + .limit(50) + .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); } From 49f1b2c64980ad5d3a384632dd315263a519ae69 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 16:02:56 +0200 Subject: [PATCH 12/46] fix: improve metadata file resolution in ZipStreamStrategy to search subdirectories --- .../ro_crate/reader/ZipStreamStrategy.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index cb4f53af..be21f6d4 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import java.io.File; import java.io.FileOutputStream; @@ -13,6 +14,7 @@ import net.lingala.zip4j.io.inputstream.ZipInputStream; import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +132,21 @@ public ObjectNode readMetadataJson(InputStream stream) { } ObjectMapper objectMapper = MyObjectMapper.getMapper(); - File jsonMetadata = temporaryFolder.resolve("ro-crate-metadata.json").toFile(); + File jsonMetadata = temporaryFolder.resolve(JsonDescriptor.ID).toFile(); + if (!jsonMetadata.isFile()) { + // Try to find the metadata file in subdirectories + File firstSubdir = FileUtils.listFilesAndDirs( + temporaryFolder.toFile(), + FileFilterUtils.directoryFileFilter(), + null + ) + .stream() + .limit(50) + .filter(file -> file.toPath().toAbsolutePath().resolve(JsonDescriptor.ID).toFile().isFile()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); + jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); + } try { return objectMapper.readTree(jsonMetadata).deepCopy(); From 6f1948c5228f9a9a346503656f30f2d6fc2ae250 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 16:05:15 +0200 Subject: [PATCH 13/46] fix: handle directory entries in ZipStreamStrategy to ensure proper extraction --- .../kit/datamanager/ro_crate/reader/ZipStreamStrategy.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index be21f6d4..03286a29 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -110,6 +110,11 @@ private void readCrate(InputStream stream) { if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { throw new IOException("Entry is outside of target directory: " + fileName); } + if (localFileHeader.isDirectory()) { + FileUtils.forceMkdir(extractedFile); + continue; + } + FileUtils.forceMkdir(extractedFile.getParentFile()); try (OutputStream outputStream = new FileOutputStream(extractedFile)) { while ((readLen = zipInputStream.read(readBuffer)) != -1) { outputStream.write(readBuffer, 0, readLen); From 5dd5421d77fd165fa316084d1bfd7cc0e7698af4 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 6 May 2025 17:10:56 +0200 Subject: [PATCH 14/46] fix: integrate Apache Commons Compress for improved zip file handling in reader.ZipStreamStrategy --- build.gradle | 1 + .../ro_crate/reader/ZipStreamStrategy.java | 34 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index f7fcd613..2522dd13 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,7 @@ dependencies { implementation group: 'commons-io', name: 'commons-io', version: '2.19.0' // read from and write to zip files implementation group: 'net.lingala.zip4j', name: 'zip4j', version: '2.11.5' + implementation 'org.apache.commons:commons-compress:1.27.1' // compare json documents in tests implementation 'com.github.fslev:json-compare:7.0' // url validator diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index 03286a29..01f6504d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -4,15 +4,14 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; + +import java.io.*; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Enumeration; import java.util.UUID; -import net.lingala.zip4j.io.inputstream.ZipInputStream; -import net.lingala.zip4j.model.LocalFileHeader; + +import org.apache.commons.compress.archivers.zip.*; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; @@ -99,26 +98,27 @@ private void readCrate(InputStream stream) { FileUtils.forceMkdir(folder); } - LocalFileHeader localFileHeader; - int readLen; - byte[] readBuffer = new byte[4096]; - try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { - while ((localFileHeader = zipInputStream.getNextEntry()) != null) { - String fileName = localFileHeader.getFileName(); + ZipFile.Builder zipFileBuilder = new ZipFile.Builder() + .setByteArray(stream.readAllBytes()) + .setIgnoreLocalFileHeader(true) + .setOpenOptions(StandardOpenOption.READ); + try (ZipFile zipFile = zipFileBuilder.get()) { + Enumeration filesInZip = zipFile.getEntries(); + while (filesInZip.hasMoreElements()) { + ZipArchiveEntry entry = filesInZip.nextElement(); + String fileName = entry.getName(); File extractedFile = new File(folder, fileName).getCanonicalFile(); if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { throw new IOException("Entry is outside of target directory: " + fileName); } - if (localFileHeader.isDirectory()) { + if (entry.isDirectory()) { FileUtils.forceMkdir(extractedFile); continue; } FileUtils.forceMkdir(extractedFile.getParentFile()); try (OutputStream outputStream = new FileOutputStream(extractedFile)) { - while ((readLen = zipInputStream.read(readBuffer)) != -1) { - outputStream.write(readBuffer, 0, readLen); - } + zipFile.getInputStream(entry).transferTo(outputStream); } } } From a704a0b6a50852790a0fd8afd3248d144072467d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 12:09:44 +0200 Subject: [PATCH 15/46] Revert "fix: integrate Apache Commons Compress for improved zip file handling in reader.ZipStreamStrategy" This reverts commit 5dd5421d77fd165fa316084d1bfd7cc0e7698af4. --- build.gradle | 1 - .../ro_crate/reader/ZipStreamStrategy.java | 34 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index 2522dd13..f7fcd613 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,6 @@ dependencies { implementation group: 'commons-io', name: 'commons-io', version: '2.19.0' // read from and write to zip files implementation group: 'net.lingala.zip4j', name: 'zip4j', version: '2.11.5' - implementation 'org.apache.commons:commons-compress:1.27.1' // compare json documents in tests implementation 'com.github.fslev:json-compare:7.0' // url validator diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index 01f6504d..03286a29 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -4,14 +4,15 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; - -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Enumeration; import java.util.UUID; - -import org.apache.commons.compress.archivers.zip.*; +import net.lingala.zip4j.io.inputstream.ZipInputStream; +import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; @@ -98,27 +99,26 @@ private void readCrate(InputStream stream) { FileUtils.forceMkdir(folder); } + LocalFileHeader localFileHeader; + int readLen; + byte[] readBuffer = new byte[4096]; - ZipFile.Builder zipFileBuilder = new ZipFile.Builder() - .setByteArray(stream.readAllBytes()) - .setIgnoreLocalFileHeader(true) - .setOpenOptions(StandardOpenOption.READ); - try (ZipFile zipFile = zipFileBuilder.get()) { - Enumeration filesInZip = zipFile.getEntries(); - while (filesInZip.hasMoreElements()) { - ZipArchiveEntry entry = filesInZip.nextElement(); - String fileName = entry.getName(); + try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { + while ((localFileHeader = zipInputStream.getNextEntry()) != null) { + String fileName = localFileHeader.getFileName(); File extractedFile = new File(folder, fileName).getCanonicalFile(); if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { throw new IOException("Entry is outside of target directory: " + fileName); } - if (entry.isDirectory()) { + if (localFileHeader.isDirectory()) { FileUtils.forceMkdir(extractedFile); continue; } FileUtils.forceMkdir(extractedFile.getParentFile()); try (OutputStream outputStream = new FileOutputStream(extractedFile)) { - zipFile.getInputStream(entry).transferTo(outputStream); + while ((readLen = zipInputStream.read(readBuffer)) != -1) { + outputStream.write(readBuffer, 0, readLen); + } } } } From 586276b8544d3324877462ee3a43c930a5ddaedc Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 12:44:42 +0200 Subject: [PATCH 16/46] Revert "fix: invalid entities from our providers should be logged and not written" This reverts commit 86654f65fbb5dcaf1f6df75ace43524c93331ed4. --- .../externalproviders/dataentities/ImportFromZenodo.java | 4 ++-- .../externalproviders/personprovider/OrcidProvider.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java index 2cc6680a..4a99ee8a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java @@ -112,12 +112,12 @@ private static void addToCrateFromZotero(String url, Crate crate) { for (var entity : graph) { if (entity.get("@id").asText().equals(mainId)) { var dataEntity = new DataEntity.DataEntityBuilder() - .setAllIfValid((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addDataEntity(dataEntity); } else { // here we have to think of a way to differentiate between data and contextual entities. var contextualEntity = new ContextualEntity.ContextualEntityBuilder() - .setAllIfValid((ObjectNode) entity).build(); + .setAllUnsafe((ObjectNode) entity).build(); crate.addContextualEntity(contextualEntity); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java index dd0b802b..17207993 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/personprovider/OrcidProvider.java @@ -72,7 +72,7 @@ public static PersonEntity getPerson(String url) { node.set(element.getKey(), element.getValue()); } } - return new PersonEntity.PersonEntityBuilder().setAllIfValid(node).build(); + return new PersonEntity.PersonEntityBuilder().setAllUnsafe(node).build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); logger.error(errorMessage); From eac3a7591326944ce766da5db14c36e803d0bcd3 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 13:09:39 +0200 Subject: [PATCH 17/46] fix: propagate IOException in crate reading and writing methods --- .../edu/kit/datamanager/ro_crate/reader/CrateReader.java | 3 ++- .../datamanager/ro_crate/reader/GenericReaderStrategy.java | 5 +++-- .../edu/kit/datamanager/ro_crate/writer/CrateWriter.java | 4 +++- .../datamanager/ro_crate/writer/GenericWriterStrategy.java | 4 +++- .../kit/datamanager/ro_crate/crate/BuilderSpec12Test.java | 3 ++- .../kit/datamanager/ro_crate/crate/ReadAndWriteTest.java | 2 +- .../datamanager/ro_crate/crate/TestRemoveAddContext.java | 3 ++- .../ro_crate/crate/preview/PreviewCrateTest.java | 6 +++--- .../kit/datamanager/ro_crate/reader/FolderReaderTest.java | 2 +- .../ro_crate/reader/RoCrateReaderSpec12Test.java | 3 ++- .../edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java | 2 +- 11 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 6586f1ed..7dcadbdc 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -83,7 +84,7 @@ public CrateReader(GenericReaderStrategy strategy) { * @param location the location of the ro-crate to be read * @return the read RO-crate */ - public RoCrate readCrate(T location) { + public RoCrate readCrate(T location) throws IOException { // get the ro-crate-metadata.json ObjectNode metadataJson = strategy.readMetadataJson(location); // get the content of the crate diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java index c3539b17..03f6b9c9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.File; +import java.io.IOException; /** * Generic interface for the strategy of the reader class. @@ -16,7 +17,7 @@ public interface GenericReaderStrategy { * @param location the location to read from * @return the parsed metadata.json as ObjectNode */ - ObjectNode readMetadataJson(T location); + ObjectNode readMetadataJson(T location) throws IOException; /** * Read the content from the given location. @@ -24,5 +25,5 @@ public interface GenericReaderStrategy { * @param location the location to read from * @return the content as a File */ - File readContent(T location); + File readContent(T location) throws IOException; } \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java index 440be0c4..4ee1be6b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -4,6 +4,8 @@ import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; +import java.io.IOException; + /** * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) @@ -22,7 +24,7 @@ public CrateWriter(GenericWriterStrategy strategy) { * @param crate the crate to write. * @param destination the location where the crate should be written. */ - public void save(Crate crate, DESTINATION destination) { + public void save(Crate crate, DESTINATION destination) throws IOException { Validator defaultValidation = new Validator(new JsonSchemaValidation()); defaultValidation.validate(crate); this.strategy.save(crate, destination); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java index 6306b576..c0e79779 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java @@ -2,6 +2,8 @@ import edu.kit.datamanager.ro_crate.Crate; +import java.io.IOException; + /** * Generic interface for the strategy of the writer class. * This allows for flexible output types when implementing different writing strategies. @@ -15,5 +17,5 @@ public interface GenericWriterStrategy { * @param crate The crate to save * @param destination The destination where the crate should be saved */ - void save(Crate crate, DESTINATION destination); + void save(Crate crate, DESTINATION destination) throws IOException; } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java index eddd93ee..270127af 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; @@ -37,7 +38,7 @@ void testAppendConformsTo() throws URISyntaxException { } @Test - void testModificationOfDraftCrate() throws URISyntaxException { + void testModificationOfDraftCrate() throws URISyntaxException, IOException { String path = this.getClass().getResource("/crates/spec-1.2-DRAFT/minimal-with-conformsTo-Array").getPath(); RoCrate crate = Readers.newFolderReader().readCrate(path); Collection existingProfiles = crate.getProfiles(); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java index 3c2e49b5..ca742ded 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java @@ -49,7 +49,7 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { @SuppressWarnings("DataFlowIssue") @Test - void testReadCrateWithHasPartHierarchy() { + void testReadCrateWithHasPartHierarchy() throws IOException { CrateReader reader = Readers.newFolderReader(); RoCrate crate = reader.readCrate(ReadAndWriteTest.class.getResource("/crates/hasPartHierarchy").getPath()); assertEquals(1, crate.getAllContextualEntities().size()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java index b1f6e21b..31c0dc8e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; import java.util.Objects; import java.util.Set; @@ -17,7 +18,7 @@ public class TestRemoveAddContext { private RoCrate crateWithComplexContext; @BeforeEach - void setup() { + void setup() throws IOException { String crateManifestPath = "/crates/extendedContextExample/"; crateManifestPath = Objects.requireNonNull(TestRemoveAddContext.class.getResource(crateManifestPath)).getPath(); this.crateWithComplexContext = Readers.newFolderReader().readCrate(crateManifestPath); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index 1ecc2062..92d70449 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -22,7 +22,7 @@ public class PreviewCrateTest { @Test - void testAutomaticPreview(@TempDir Path temp) { + void testAutomaticPreview(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate1"); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new AutomaticPreview()) @@ -33,7 +33,7 @@ void testAutomaticPreview(@TempDir Path temp) { } @Test - void testAutomaticPreviewAddingLater(@TempDir Path temp) { + void testAutomaticPreviewAddingLater(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate2"); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(null)//disable preview to allow to compare folders before and after @@ -47,7 +47,7 @@ void testAutomaticPreviewAddingLater(@TempDir Path temp) { } @Test - void testCustomPreview(@TempDir Path temp) { + void testCustomPreview(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate1"); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new CustomPreview()) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index a7143862..8375ba36 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -20,7 +20,7 @@ class FolderReaderTest implements CommonReaderTest { @Override - public void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newFolderWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isDirectory()); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java index 7b1df7df..68094231 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.util.stream.StreamSupport; import org.junit.jupiter.api.Test; @@ -27,7 +28,7 @@ public class RoCrateReaderSpec12Test { * https://www.researchobject.org/ro-crate/1.2-DRAFT/profiles.html#declaring-conformance-of-an-ro-crate-profile */ @Test - void testReadingCrateWithConformsToArray() { + void testReadingCrateWithConformsToArray() throws IOException { String path = this.getClass().getResource("/crates/spec-1.2-DRAFT/minimal-with-conformsTo-Array").getPath(); Crate crate = Readers.newFolderReader().readCrate(path); JsonNode conformsTo = crate.getJsonDescriptor().getProperty("conformsTo"); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index a4616f8e..7d71d94e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -13,7 +13,7 @@ class ZipReaderTest implements ElnFileFormatTest { @Override - public void saveCrate(Crate crate, Path target) { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); assertTrue(target.toFile().isFile()); } From 62b1b0c63dc55aede79fd433db85a277717832b6 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 13:10:30 +0200 Subject: [PATCH 18/46] refactor: setAll method now uses setAllIfValid to avoid code duplication --- .../kit/datamanager/ro_crate/entities/AbstractEntity.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java index 3ec7bb9f..34889d21 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java @@ -572,11 +572,7 @@ public T addIdFromCollectionOfEntities(String name, Collection e */ @Deprecated(since = "2.1.0", forRemoval = true) public T setAll(ObjectNode properties) { - if (AbstractEntity.entityValidation.entityValidation(properties)) { - this.properties = properties; - this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties)); - } - return self(); + return setAllIfValid(properties); } /** From d3f798692a13901fd44eaf2d356573e2ac2a87cb Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 13:30:36 +0200 Subject: [PATCH 19/46] tests: add blacklist functionality for unsupported ELN files in tests --- .../ro_crate/reader/ZipStreamStrategy.java | 80 ++++++++----------- .../ro_crate/reader/ElnFileFormatTest.java | 27 ++++++- .../ro_crate/reader/ZipStreamReaderTest.java | 14 ++++ 3 files changed, 72 insertions(+), 49 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index 03286a29..f6ed4577 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -85,53 +85,49 @@ public boolean isExtracted() { * * @param stream The input stream. */ - private void readCrate(InputStream stream) { - try { - File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.exists()) { - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - } else { - FileUtils.forceMkdir(folder); + private void readCrate(InputStream stream) throws IOException { + File folder = temporaryFolder.toFile(); + // ensure the directory is clean + if (folder.exists()) { + if (folder.isDirectory()) { + FileUtils.cleanDirectory(folder); + } else if (folder.isFile()) { + FileUtils.delete(folder); } + } else { + FileUtils.forceMkdir(folder); + } - LocalFileHeader localFileHeader; - int readLen; - byte[] readBuffer = new byte[4096]; + LocalFileHeader localFileHeader; + int readLen; + byte[] readBuffer = new byte[4096]; - try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { - while ((localFileHeader = zipInputStream.getNextEntry()) != null) { - String fileName = localFileHeader.getFileName(); - File extractedFile = new File(folder, fileName).getCanonicalFile(); - if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { - throw new IOException("Entry is outside of target directory: " + fileName); - } - if (localFileHeader.isDirectory()) { - FileUtils.forceMkdir(extractedFile); - continue; - } - FileUtils.forceMkdir(extractedFile.getParentFile()); - try (OutputStream outputStream = new FileOutputStream(extractedFile)) { - while ((readLen = zipInputStream.read(readBuffer)) != -1) { - outputStream.write(readBuffer, 0, readLen); - } + try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { + while ((localFileHeader = zipInputStream.getNextEntry()) != null) { + String fileName = localFileHeader.getFileName(); + File extractedFile = new File(folder, fileName).getCanonicalFile(); + if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { + throw new IOException("Entry is outside of target directory: " + fileName); + } + if (localFileHeader.isDirectory()) { + FileUtils.forceMkdir(extractedFile); + continue; + } + FileUtils.forceMkdir(extractedFile.getParentFile()); + try (OutputStream outputStream = new FileOutputStream(extractedFile)) { + while ((readLen = zipInputStream.read(readBuffer)) != -1) { + outputStream.write(readBuffer, 0, readLen); } } } - this.isExtracted = true; - // register deletion on exit - FileUtils.forceDeleteOnExit(folder); - } catch (IOException ex) { - logger.error("Failed to read crate from input stream.", ex); } + this.isExtracted = true; + // register deletion on exit + FileUtils.forceDeleteOnExit(folder); } @Override - public ObjectNode readMetadataJson(InputStream stream) { + public ObjectNode readMetadataJson(InputStream stream) throws IOException { if (!isExtracted) { this.readCrate(stream); } @@ -152,17 +148,11 @@ public ObjectNode readMetadataJson(InputStream stream) { .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); } - - try { - return objectMapper.readTree(jsonMetadata).deepCopy(); - } catch (IOException e) { - logger.error("Failed to deserialize crate metadata.", e); - return null; - } + return objectMapper.readTree(jsonMetadata).deepCopy(); } @Override - public File readContent(InputStream stream) { + public File readContent(InputStream stream) throws IOException { if (!isExtracted) { this.readCrate(stream); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java index 3e06fc7f..bc10848c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ElnFileFormatTest.java @@ -19,6 +19,20 @@ public interface ElnFileFormatTest< > extends TestableReaderStrategy { + /** + * Some readers may not be able to read a subset of eln files, + * e.g. because a zip file may not be readable in streaming mode. + *

+ * An implementation test may use this methode to provide a subset of the + * test cases where an IOException is expected. + * + * @param input the input to test for presence in the blacklist + * @return true if the input is in the blacklist, false otherwise + */ + default boolean isInBlacklist(String input) { + return false; + } + /** * ELN Crates are zip files not fully compatible with the Ro-Crate standard * in the sense that they must contain a single subfolder in the zip file @@ -46,9 +60,14 @@ default void testReadElnCrates(String urlStr, @TempDir Path tmp) throws IOExcept FileUtils.copyURLToFile(url, elnFile.toFile(), 10000, 10000); assertTrue(elnFile.toFile().exists()); - // Read the crate from the downloaded file - Crate read = this.readCrate(elnFile); - assertNotNull(read); - assertFalse(read.getAllDataEntities().isEmpty()); + if (!isInBlacklist(urlStr)) { + // Read the crate from the downloaded file + Crate read = this.readCrate(elnFile); + assertNotNull(read); + assertFalse(read.getAllDataEntities().isEmpty()); + } else { + // If the file is in the blacklist, we expect an IOException + assertThrows(IOException.class, () -> this.readCrate(elnFile)); + } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index 17b37247..a6cca8e6 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -5,6 +5,7 @@ import java.io.*; import java.nio.file.Path; +import java.util.Set; import static org.junit.jupiter.api.Assertions.*; @@ -13,6 +14,19 @@ class ZipStreamReaderTest implements CommonReaderTest, ElnFileFormatTest { + /** + * At the point of writing this test, + * these files are in a zip format which cannot be read in streaming mode + */ + @Override + public boolean isInBlacklist(String input) { + return Set.of( + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/records-example.eln", + "https://github.com/TheELNConsortium/TheELNFileFormat/raw/refs/heads/master/examples/kadi4mat/collections-example.eln" + ) + .contains(input); + } + @Override public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); From cca2cd5672eb57c8825126d42a325dbd0b25602d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:18:06 +0200 Subject: [PATCH 20/46] tests: remove possible side effects between tests in LearnByExampleTest.java --- .../ro_crate/examples/LearnByExampleTest.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java index aab61632..64733e89 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java @@ -42,20 +42,22 @@ public class LearnByExampleTest { /** * This creates a valid, empty RO-Crate builder. */ - static final RoCrate.RoCrateBuilder STARTER_CRATE = new RoCrate.RoCrateBuilder( - "name", - "description", - "2025", - "licenseIdentifier" - ); + static RoCrate.RoCrateBuilder NEW_STARTER_CRATE() { + return new RoCrate.RoCrateBuilder( + "name", + "description", + "2025", + "licenseIdentifier" + ); + } /** * Calling the `build()` method on the builder creates a valid RO-Crate. - * Run this test to view the STARTER_CRATE JSON in the console. + * Run this test to view the NEW_STARTER_CRATE() JSON in the console. */ @Test void aSimpleCrate() { - RoCrate almostEmptyCrate = STARTER_CRATE.build(); + RoCrate almostEmptyCrate = NEW_STARTER_CRATE().build(); assertNotNull(almostEmptyCrate); HelpFunctions.prettyPrintJsonString(almostEmptyCrate.getJsonMetadata()); } @@ -72,7 +74,7 @@ void aSimpleCrate() { */ @Test void addingYourFirstEntity() { - RoCrate myFirstCrate = STARTER_CRATE + RoCrate myFirstCrate = NEW_STARTER_CRATE() // We can add new terms to our crate. The terms we can use are called "context". .addValuePairToContext("Station", "www.station.com") // We can also add whole contexts to our crate. @@ -115,7 +117,7 @@ void addingYourFirstEntity() { */ @Test void specializingYourFirstEntity() { - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addDataEntity( // Let's do something custom: new DataEntity.DataEntityBuilder() @@ -148,7 +150,7 @@ void referencingFilesOnTheWeb() { // Let's say this is the file we would like to point at with an entity. String lovelyFile = "https://github.com/kit-data-manager/ro-crate-java/issues/5"; - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addDataEntity( // Build our entity to point to the file: new FileEntity.FileEntityBuilder() @@ -187,7 +189,7 @@ void includingFilesIntoTheCrateFolder(@TempDir Path tempDir) throws IOException // But in the crate we want it to be String seriousExperimentFile = "fantastic-experiment/2025-01-01.csv"; - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addDataEntity( // Build our entity to point to the file: new FileEntity.FileEntityBuilder() @@ -236,7 +238,7 @@ void addingContextualEntities() { PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addContextualEntity(person) .addContextualEntity(organization) .build(); @@ -263,7 +265,7 @@ void writingAndReadingCrates(@TempDir Path tempDir) throws IOException { PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addContextualEntity(person) .addContextualEntity(organization) .build(); @@ -324,7 +326,7 @@ void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-6575-1022"); OrganizationEntity organization = RorProvider.getOrganization("https://ror.org/04t3en479"); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .addContextualEntity(person) .addContextualEntity(organization) .build(); @@ -368,7 +370,7 @@ void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { */ @Test void humanReadableContent() { - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .setPreview(new AutomaticPreview()) .build(); @@ -380,11 +382,13 @@ void humanReadableContent() { * Therefore, the constructor is a bit more complicated. */ @Test - void staticPreview(@TempDir Path tempDir) { + void staticPreview(@TempDir Path tempDir) throws IOException { File mainPreviewHtml = tempDir.resolve("mainPreview.html").toFile(); File additionalFilesDirectory = tempDir.resolve("additionalFiles").toFile(); + FileUtils.forceMkdir(additionalFilesDirectory); + FileUtils.touch(mainPreviewHtml); - RoCrate crate = STARTER_CRATE + RoCrate crate = NEW_STARTER_CRATE() .setPreview(new StaticPreview(mainPreviewHtml, additionalFilesDirectory)) .build(); @@ -408,7 +412,7 @@ void validation() { String schemaPath = schemaUrl.getPath(); // This crate for sure is not a workflow, so validation will fail. - RoCrate crate = STARTER_CRATE.build(); + RoCrate crate = NEW_STARTER_CRATE().build(); // And now do the validation. Validator validator = new Validator(new JsonSchemaValidation(schemaPath)); From 6fd9ac26fb19d8cb457fde4eda4ae5d4b176532a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:35:05 +0200 Subject: [PATCH 21/46] refactor: pull DataEntity saveToStream logic into writer.ZipStreamStrategy The zip dependency is too widely spread over different classes. We use it for reading and writing, so it should stay in classes which handle reading and writing. --- .../ro_crate/entities/data/DataEntity.java | 17 ---------- .../ro_crate/entities/data/DataSetEntity.java | 13 -------- .../ro_crate/writer/ZipStreamStrategy.java | 33 ++++++++++++++++++- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index e6e28f8f..28d043c9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -5,7 +5,6 @@ import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl; -import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; import java.io.IOException; @@ -15,7 +14,6 @@ import java.util.List; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -72,21 +70,6 @@ public void saveToZip(ZipFile zipFile) throws ZipException { } } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a zip archive. - * - * @param zipStream The zip output stream where it should be written. - * @throws ZipException when something goes wrong with the writing to the - * zip file. - * @throws IOException If opening the file input stream fails. - */ - public void saveToStream(ZipOutputStream zipStream) throws ZipException, IOException { - if (this.path != null) { - ZipUtil.addFileToZipStream(zipStream, this.path.toFile(), this.getId()); - } - } - /** * If the data entity contains a physical file. This method will write it * when the crate is being written to a folder. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index 832d9819..7ea4c789 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -4,14 +4,11 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import edu.kit.datamanager.ro_crate.entities.serializers.HasPartSerializer; -import edu.kit.datamanager.ro_crate.util.ZipUtil; -import java.io.IOException; import java.util.HashSet; import java.util.Set; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; /** @@ -53,16 +50,6 @@ public void saveToZip(ZipFile zipFile) throws ZipException { } } - @Override - public void saveToStream(ZipOutputStream zipOutputStream) throws IOException { - if (this.getPath() != null) { - ZipUtil.addFolderToZipStream( - zipOutputStream, - this.getPath().toAbsolutePath().toString(), - this.getId()); - } - } - public void addToHasPart(String id) { this.hasPart.add(id); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index 818c064c..2a9d1fbb 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -12,6 +12,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; + +import edu.kit.datamanager.ro_crate.util.ZipUtil; +import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.slf4j.Logger; @@ -39,7 +42,7 @@ public void save(Crate crate, OutputStream destination) { private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { for (DataEntity dataEntity : crate.getAllDataEntities()) { try { - dataEntity.saveToStream(zipStream); + saveToStream(dataEntity, zipStream); } catch (IOException e) { logger.error("Could not save {} to zip stream!", dataEntity.getId(), e); } @@ -74,4 +77,32 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { logger.error("Exception writing ro-crate-metadata.json file to zip.", e); } } + + /** + * If the data entity contains a physical file. This method will write it + * when the crate is being written to a zip archive. + * + * @param zipStream The zip output stream where it should be written. + * @throws ZipException when something goes wrong with the writing to the + * zip file. + * @throws IOException If opening the file input stream fails. + */ + private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws ZipException, IOException { + if (entity == null) { + return; + } + + boolean isDirectory = entity.getPath().toFile().isDirectory(); + if (isDirectory) { + ZipUtil.addFolderToZipStream( + zipStream, + entity.getPath().toAbsolutePath().toString(), + entity.getId()); + } else { + ZipUtil.addFileToZipStream( + zipStream, + entity.getPath().toFile(), + entity.getId()); + } + } } From b58d6bdd9d054d277808e9890f9dfcc6562dc780 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:38:38 +0200 Subject: [PATCH 22/46] refactor: propagate IOExceptions properly in writer.ZipStreamStrategy.java --- .../ro_crate/writer/ZipStreamStrategy.java | 57 ++++++++----------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index 2a9d1fbb..db69cb3a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -29,52 +29,41 @@ public class ZipStreamStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); @Override - public void save(Crate crate, OutputStream destination) { + public void save(Crate crate, OutputStream destination) throws IOException { try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { saveMetadataJson(crate, zipFile); saveDataEntities(crate, zipFile); - } catch (IOException e) { - // can not close ZipOutputStream (threw Exception) - logger.error("Failed to save ro-crate to zip stream.", e); } } - private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { + private void saveDataEntities(Crate crate, ZipOutputStream zipStream) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - saveToStream(dataEntity, zipStream); - } catch (IOException e) { - logger.error("Could not save {} to zip stream!", dataEntity.getId(), e); - } + saveToStream(dataEntity, zipStream); } } - private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { - try { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - // write the ro-crate-metadata + private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOException { + // write the metadata.json file + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + // write the ro-crate-metadata - byte[] buff = new byte[4096]; - int readLen; - zipStream.putNextEntry(zipParameters); - try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { - while ((readLen = inputStream.read(buff)) != -1) { - zipStream.write(buff, 0, readLen); - } + byte[] buff = new byte[4096]; + int readLen; + zipStream.putNextEntry(zipParameters); + try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { + while ((readLen = inputStream.read(buff)) != -1) { + zipStream.write(buff, 0, readLen); } - zipStream.closeEntry(); + } + zipStream.closeEntry(); - if (crate.getPreview() != null) { - crate.getPreview().saveAllToStream(str, zipStream); - } - } catch (IOException e) { - logger.error("Exception writing ro-crate-metadata.json file to zip.", e); + if (crate.getPreview() != null) { + crate.getPreview().saveAllToStream(str, zipStream); } } @@ -87,7 +76,7 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { * zip file. * @throws IOException If opening the file input stream fails. */ - private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws ZipException, IOException { + private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws IOException { if (entity == null) { return; } From 6e1dae62955c437b824981319066dd8fd44537cf Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:40:41 +0200 Subject: [PATCH 23/46] refactor: propagate IOExceptions properly in writer.FolderStrategy --- .../ro_crate/writer/FolderStrategy.java | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index b2585637..6a3c85c0 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -26,38 +26,30 @@ public class FolderStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); @Override - public void save(Crate crate, String destination) { + public void save(Crate crate, String destination) throws IOException { File file = new File(destination); - try { - FileUtils.forceMkdir(file); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + FileUtils.forceMkdir(file); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); - File json = new File(destination, "ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(inputStream, json); - inputStream.close(); - // save also the preview files to the crate destination - if (crate.getPreview() != null) { - crate.getPreview().saveAllToFolder(file); - } - for (var e : crate.getUntrackedFiles()) { - if (e.isDirectory()) { - FileUtils.copyDirectoryToDirectory(e, file); - } else { - FileUtils.copyFileToDirectory(e, file); - } + File json = new File(destination, "ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(inputStream, json); + inputStream.close(); + // save also the preview files to the crate destination + if (crate.getPreview() != null) { + crate.getPreview().saveAllToFolder(file); + } + for (var e : crate.getUntrackedFiles()) { + if (e.isDirectory()) { + FileUtils.copyDirectoryToDirectory(e, file); + } else { + FileUtils.copyFileToDirectory(e, file); } - } catch (IOException e) { - logger.error("Error creating destination directory!", e); } for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.savetoFile(file); - } catch (IOException e) { - logger.error("Cannot save " + dataEntity.getId() + " to destination folder!", e); - } + dataEntity.savetoFile(file); } } } From 82cee04865cd23f9cddd2395e396b9286eb39820 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 15:42:21 +0200 Subject: [PATCH 24/46] refactor: propagate IOExceptions properly in writer.ZipStrategy --- .../ro_crate/writer/ZipStrategy.java | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index 6a11df87..00b920d2 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -25,44 +25,33 @@ public class ZipStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); @Override - public void save(Crate crate, String destination) { + public void save(Crate crate, String destination) throws IOException { try (ZipFile zipFile = new ZipFile(destination)) { saveMetadataJson(crate, zipFile); saveDataEntities(crate, zipFile); - } catch (IOException e) { - // can not close ZipFile (threw Exception) - logger.error("Failed to write ro-crate to destination " + destination + ".", e); } } - private void saveDataEntities(Crate crate, ZipFile zipFile) { + private void saveDataEntities(Crate crate, ZipFile zipFile) throws ZipException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.saveToZip(zipFile); - } catch (ZipException e) { - logger.error("Could not save " + dataEntity.getId() + " to zip file!", e); - } + dataEntity.saveToZip(zipFile); } } - private void saveMetadataJson(Crate crate, ZipFile zipFile) { + private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { // write the metadata.json file ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); - try { - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - } - if (crate.getPreview() != null) { - crate.getPreview().saveAllToZip(zipFile); - } - } catch (IOException e) { - logger.error("Exception writing ro-crate-metadata.json file to zip.", e); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { + // write the ro-crate-metadata + zipFile.addStream(inputStream, zipParameters); + } + if (crate.getPreview() != null) { + crate.getPreview().saveAllToZip(zipFile); } } } From bee2917fedd32822f2720632dc55177c04c4e22b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:26:30 +0200 Subject: [PATCH 25/46] refactor: pull DataEntity saveToZip logic into writer.ZipStrategy --- .../ro_crate/entities/data/DataEntity.java | 19 --------------- .../ro_crate/entities/data/DataSetEntity.java | 13 ----------- .../ro_crate/writer/ZipStrategy.java | 23 ++++++++++++++++--- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 28d043c9..be4253b1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -12,9 +12,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; /** @@ -54,22 +51,6 @@ public void addAuthorId(String id) { this.addIdProperty("author", id); } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a zip archive. - * - * @param zipFile the zipFile where it should be written. - * @throws ZipException when something goes wrong with the writing to the - * zip file. - */ - public void saveToZip(ZipFile zipFile) throws ZipException { - if (this.path != null) { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(this.getId()); - zipFile.addFile(this.path.toFile(), zipParameters); - } - } - /** * If the data entity contains a physical file. This method will write it * when the crate is being written to a folder. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index 7ea4c789..2ef078ff 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -7,9 +7,6 @@ import java.util.HashSet; import java.util.Set; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; /** * A helping class for the creating of Data entities of type Dataset. @@ -40,16 +37,6 @@ public void removeFromHasPart(String str) { this.hasPart.remove(str); } - @Override - public void saveToZip(ZipFile zipFile) throws ZipException { - if (this.getPath() != null) { - ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(this.getId()); - parameters.setIncludeRootFolder(false); - zipFile.addFolder(this.getPath().toFile(), parameters); - } - } - public void addToHasPart(String id) { this.hasPart.add(id); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index 00b920d2..0c8634b1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -6,7 +6,6 @@ import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.model.ZipParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,9 +31,9 @@ public void save(Crate crate, String destination) throws IOException { } } - private void saveDataEntities(Crate crate, ZipFile zipFile) throws ZipException { + private void saveDataEntities(Crate crate, ZipFile zipFile) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - dataEntity.saveToZip(zipFile); + this.saveToZip(dataEntity, zipFile); } } @@ -54,4 +53,22 @@ private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { crate.getPreview().saveAllToZip(zipFile); } } + + private void saveToZip(DataEntity entity, ZipFile zipFile) throws IOException { + if (entity == null || entity.getPath() == null) { + return; + } + + boolean isDirectory = entity.getPath().toFile().isDirectory(); + if (isDirectory) { + ZipParameters parameters = new ZipParameters(); + parameters.setRootFolderNameInZip(entity.getId()); + parameters.setIncludeRootFolder(false); + zipFile.addFolder(entity.getPath().toFile(), parameters); + } else { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(entity.getId()); + zipFile.addFile(entity.getPath().toFile(), zipParameters); + } + } } From 08a5f10a47c23e3b6cb3e3b50745cef358ced51c Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:27:07 +0200 Subject: [PATCH 26/46] refactor: remove outdated saveToStream method documentation in writer.ZipStreamStrategy --- .../datamanager/ro_crate/writer/ZipStreamStrategy.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index db69cb3a..b97b3c2a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -67,15 +67,6 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOE } } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a zip archive. - * - * @param zipStream The zip output stream where it should be written. - * @throws ZipException when something goes wrong with the writing to the - * zip file. - * @throws IOException If opening the file input stream fails. - */ private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws IOException { if (entity == null) { return; From 0fc94e78a3718c39e18b8f4986580e2abdb9177b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:33:32 +0200 Subject: [PATCH 27/46] refactor: move savetoFile logic from DataEntity to FolderStrategy --- .../ro_crate/entities/data/DataEntity.java | 20 ------------------- .../ro_crate/writer/FolderStrategy.java | 12 ++++++++++- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index be4253b1..864d3044 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -6,13 +6,10 @@ import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl; -import java.io.File; -import java.io.IOException; import java.net.URI; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import org.apache.commons.io.FileUtils; /** * The base class of every data entity. @@ -51,23 +48,6 @@ public void addAuthorId(String id) { this.addIdProperty("author", id); } - /** - * If the data entity contains a physical file. This method will write it - * when the crate is being written to a folder. - * - * @param file the folder location where the entity should be written. - * @throws IOException if something goes wrong with the writing. - */ - public void savetoFile(File file) throws IOException { - if (this.getPath() != null) { - if (this.getPath().toFile().isDirectory()) { - FileUtils.copyDirectory(this.getPath().toFile(), file.toPath().resolve(this.getId()).toFile()); - } else { - FileUtils.copyFile(this.getPath().toFile(), file.toPath().resolve(this.getId()).toFile()); - } - } - } - @JsonIgnore public Path getPath() { return path; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index 6a3c85c0..8fa5a6bb 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -49,7 +49,17 @@ public void save(Crate crate, String destination) throws IOException { } } for (DataEntity dataEntity : crate.getAllDataEntities()) { - dataEntity.savetoFile(file); + savetoFile(dataEntity, file); + } + } + + private void savetoFile(DataEntity entity, File file) throws IOException { + if (entity.getPath() != null) { + if (entity.getPath().toFile().isDirectory()) { + FileUtils.copyDirectory(entity.getPath().toFile(), file.toPath().resolve(entity.getId()).toFile()); + } else { + FileUtils.copyFile(entity.getPath().toFile(), file.toPath().resolve(entity.getId()).toFile()); + } } } } From e41cfcfba32f62f8025706cb2d246a607ae2b4e6 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 8 May 2025 16:37:53 +0200 Subject: [PATCH 28/46] refactor: propagate IOException handling in reader.FolderStrategy and reader.ZipStrategy --- .../ro_crate/reader/FolderStrategy.java | 8 +--- .../ro_crate/reader/ZipStrategy.java | 45 ++++++++----------- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java index f6e235ed..4b07e01d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java @@ -17,15 +17,11 @@ public class FolderStrategy implements GenericReaderStrategy { @Override - public ObjectNode readMetadataJson(String location) { + public ObjectNode readMetadataJson(String location) throws IOException { Path metadata = new File(location).toPath().resolve("ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); ObjectNode objectNode = objectMapper.createObjectNode(); - try { - objectNode = objectMapper.readTree(metadata.toFile()).deepCopy(); - } catch (IOException e) { - e.printStackTrace(); - } + objectNode = objectMapper.readTree(metadata.toFile()).deepCopy(); return objectNode; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java index 74622b2f..d25a235e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -80,29 +80,25 @@ public boolean isExtracted() { return isExtracted; } - private void readCrate(String location) { - try { - File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - // extract - try (ZipFile zf = new ZipFile(location)) { - zf.extractAll(temporaryFolder.toAbsolutePath().toString()); - this.isExtracted = true; - } - // register deletion on exit - FileUtils.forceDeleteOnExit(folder); - } catch (IOException e) { - e.printStackTrace(); + private void readCrate(String location) throws IOException { + File folder = temporaryFolder.toFile(); + // ensure the directory is clean + if (folder.isDirectory()) { + FileUtils.cleanDirectory(folder); + } else if (folder.isFile()) { + FileUtils.delete(folder); } + // extract + try (ZipFile zf = new ZipFile(location)) { + zf.extractAll(temporaryFolder.toAbsolutePath().toString()); + this.isExtracted = true; + } + // register deletion on exit + FileUtils.forceDeleteOnExit(folder); } @Override - public ObjectNode readMetadataJson(String location) { + public ObjectNode readMetadataJson(String location) throws IOException { if (!isExtracted) { this.readCrate(location); } @@ -123,17 +119,12 @@ public ObjectNode readMetadataJson(String location) { .orElseThrow(() -> new IllegalStateException("No %s found in zip file".formatted(JsonDescriptor.ID))); jsonMetadata = firstSubdir.toPath().resolve(JsonDescriptor.ID).toFile(); } - - try { - return objectMapper.readTree(jsonMetadata).deepCopy(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } + + return objectMapper.readTree(jsonMetadata).deepCopy(); } @Override - public File readContent(String location) { + public File readContent(String location) throws IOException { if (!isExtracted) { this.readCrate(location); } From bccbf0bb5fd0a266b6e99749297ad8a7333f3e96 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 17:49:27 +0200 Subject: [PATCH 29/46] tests: implement ELN style crate writer test for ZipWriterTest --- .../datamanager/ro_crate/HelpFunctions.java | 23 +++ ...eWriterTest.java => CommonWriterTest.java} | 159 ++---------------- .../ro_crate/writer/ElnFileFormatTest.java | 70 ++++++++ .../ro_crate/writer/FolderWriterTest.java | 6 +- .../writer/TestableWriterStrategy.java | 121 +++++++++++++ .../writer/ZipStreamStrategyTest.java | 4 +- .../ro_crate/writer/ZipWriterTest.java | 13 +- 7 files changed, 246 insertions(+), 150 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{CrateWriterTest.java => CommonWriterTest.java} (50%) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java diff --git a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java index 2c733d3e..24a57a07 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java @@ -17,6 +17,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -152,4 +154,25 @@ public static boolean compareTwoDir(File dir1, File dir2) throws IOException { } return true; } + + /** + * Prints the file tree of the given directory for debugging and understanding + * a test more quickly. + * + * @param directoryToPrint the directory to print + * @throws IOException if an error occurs while printing the file tree + */ + @SuppressWarnings("resource") + public static void printFileTree(Path directoryToPrint) throws IOException { + // Print all files recursively in a tree structure for debugging + System.out.printf("Files in %s:%n", directoryToPrint.getFileName().toString()); + Files.walk(directoryToPrint) + .forEach(path -> { + if (!path.toAbsolutePath().equals(directoryToPrint.toAbsolutePath())) { + int depth = path.relativize(directoryToPrint).getNameCount(); + String prefix = " ".repeat(depth); + System.out.printf("%s%s%s%n", prefix, "└── ", path.getFileName()); + } + }); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java similarity index 50% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java index 429fc450..0eae96c2 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java @@ -1,20 +1,14 @@ package edu.kit.datamanager.ro_crate.writer; -import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; -import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; -import net.lingala.zip4j.ZipFile; + import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.IOException; -import java.io.InputStream; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -22,16 +16,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -abstract class CrateWriterTest { - - /** - * Saves the crate with the writer fitting to this test class. - * - * @param crate the crate to save - * @param target the target path to the save location - * @throws IOException if an error occurs while saving the crate - */ - abstract protected void saveCrate(Crate crate, Path target) throws IOException; +interface CommonWriterTest extends TestableWriterStrategy { /** * Test where the writer needs to rename files or folders in order to make a valid crate. @@ -41,11 +26,11 @@ abstract class CrateWriterTest { * @throws IOException if an error occurs while writing the crate */ @Test - void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { + default void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Path correctCrate = tempDir.resolve("compare_with_me"); Path pathToFile = correctCrate.resolve("you-will-need-to-rename-this-file.ai"); Path pathToDir = correctCrate.resolve("you-will-need-to-rename-this-dir"); - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + createManualCrateStructure(correctCrate, pathToFile, pathToDir); Path writtenCrate = tempDir.resolve("written-crate"); Path extractionPath = tempDir.resolve("checkMe"); @@ -60,11 +45,11 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { ) .build(); this.saveCrate(builtCrate, writtenCrate); - this.ensureCrateIsExtractedIn(writtenCrate, extractionPath); + ensureCrateIsExtractedIn(writtenCrate, extractionPath); } - printFileTree(correctCrate); - printFileTree(extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); // The actual file name should **not** appear in the crate String fileName = pathToFile.getFileName().toString(); @@ -111,7 +96,7 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { * @throws IOException if an error occurs while writing the crate */ @Test - void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { + default void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { // We need a correct directory to compare with. // It is built manually to ensure we meet our expectations. // Reader-writer-consistency is tested at {@link CrateReaderTest} @@ -119,7 +104,7 @@ void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { Path pathToFile = correctCrate.resolve("cp7glop.ai"); Path pathToDir = correctCrate.resolve("lots_of_little_files"); - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + createManualCrateStructure(correctCrate, pathToFile, pathToDir); // Now use the builder to build the same crate independently. // The files will be reused (we need a place to take a copy from) @@ -130,9 +115,9 @@ void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { // extract the zip file to a temporary directory Path extractionPath = tempDir.resolve("extracted_for_testing"); - this.ensureCrateIsExtractedIn(pathToZip, extractionPath); - printFileTree(correctCrate); - printFileTree(extractionPath); + ensureCrateIsExtractedIn(pathToZip, extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); // compare the extracted directory with the correct one assertTrue(HelpFunctions.compareTwoDir( @@ -151,12 +136,12 @@ void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { * @throws IOException if an error occurs while writing the crate */ @Test - void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOException { + default void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOException { Path correctCrate = tempDir.resolve("compare_with_me"); Path pathToFile = correctCrate.resolve("cp7glop.ai"); Path pathToDir = correctCrate.resolve("lots_of_little_files"); - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + createManualCrateStructure(correctCrate, pathToFile, pathToDir); { // This file is not part of the crate, and should therefore not be present Path falseFile = correctCrate.resolve("new"); @@ -176,8 +161,8 @@ void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOExceptio // extract and compare Path extractionPath = tempDir.resolve("extracted_for_testing"); ensureCrateIsExtractedIn(pathToZip, extractionPath); - printFileTree(correctCrate); - printFileTree(extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); assertFalse(HelpFunctions.compareTwoDir( correctCrate.toFile(), @@ -187,116 +172,4 @@ void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOExceptio roCrate, "/json/crate/fileAndDir.json"); } - - /** - * Prints the file tree of the given directory for debugging and understanding - * a test more quickly. - * - * @param directoryToPrint the directory to print - * @throws IOException if an error occurs while printing the file tree - */ - @SuppressWarnings("resource") - protected static void printFileTree(Path directoryToPrint) throws IOException { - // Print all files recursively in a tree structure for debugging - System.out.printf("Files in %s:%n", directoryToPrint.getFileName().toString()); - Files.walk(directoryToPrint) - .forEach(path -> { - if (!path.toAbsolutePath().equals(directoryToPrint.toAbsolutePath())) { - int depth = path.relativize(directoryToPrint).getNameCount(); - String prefix = " ".repeat(depth); - System.out.printf("%s%s%s%n", prefix, "└── ", path.getFileName()); - } - }); - } - - /** - * Ensures the crate is in extracted form in the given path. - * - * @param pathToCrate the path to the crate, may not be a folder yet - * @param expectedPath the path where the crate should be in extracted form - * @throws IOException if an error occurs while extracting the crate - */ - protected void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { - try (ZipFile zf = new ZipFile(pathToCrate.toFile())) { - zf.extractAll(expectedPath.toFile().getAbsolutePath()); - } - } - - /** - * Creates a crate structure manually. - * - * @param correctCrate the path to the crate - * @param pathToFile the path to the file - * @param pathToDir the path to the directory - * @throws IOException if an error occurs while creating the crate structure - */ - protected void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { - FileUtils.forceMkdir(correctCrate.toFile()); - InputStream fileJson = ZipStreamStrategyTest.class - .getResourceAsStream("/json/crate/fileAndDir.json"); - Assertions.assertNotNull(fileJson); - // fill the directory with expected files and dirs - // starting with the .json of our crate - Path json = correctCrate.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - // create preview - PreviewGenerator.generatePreview(correctCrate.toFile().getAbsolutePath()); - // create the files and directories - FileUtils.writeStringToFile(pathToFile.toFile(), "content of Local File", Charset.defaultCharset()); - // creates the directory and a subdirectory - Path subdir = pathToDir.resolve("subdir"); - FileUtils.forceMkdir(subdir.toFile()); - FileUtils.writeStringToFile( - subdir.resolve("subsubfirst.txt").toFile(), - "content of subsub file in subsubdir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - pathToDir.resolve("first.txt").toFile(), - "content of first file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - pathToDir.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - pathToDir.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - } - - /** - * Creates a crate resembling the one we manually create in these tests. - * - * @param pathToFile the file to add - * @param pathToSubdir the directory to add - * @return the crate builder - */ - protected RoCrate.RoCrateBuilder getCrateWithFileAndDir(Path pathToFile, Path pathToSubdir) { - return new RoCrate.RoCrateBuilder( - "Example RO-Crate", - "The RO-Crate Root Data Entity", - "2024", - "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" - ) - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(pathToFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(pathToSubdir) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()); - } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java new file mode 100644 index 00000000..feb8dc06 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java @@ -0,0 +1,70 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +public interface ElnFileFormatTest extends TestableWriterStrategy { + + /** + * Write in ELN format style, meaning with a subfolder in the zip file. + * + * @param crate the crate to write + * @param target the target path to the save location + * @throws IOException if an error occurs + */ + void saveCrateElnStyle(Crate crate, Path target) throws IOException; + + @Test + default void testMakesElnStyleCrate(@TempDir Path tempDir) throws IOException { + // We need a correct directory to compare with. + // It is built manually to ensure we meet our expectations. + // Reader-writer-consistency is tested at {@link CrateReaderTest} + + // We compare the ELN style like this: + // tempDir + // └── compare_with_me + // └── crate-subfolder + // ├── ... + // └── extracted_for_testing + // └── crate-subfolder + // ├── ... + String crateName = "crate-subfolder"; + Path correctCrate = tempDir + .resolve("compare_with_me") + .resolve(crateName); + Path pathToFile = correctCrate.resolve("cp7glop.ai"); + Path pathToDir = correctCrate.resolve("lots_of_little_files"); + + createManualCrateStructure(correctCrate, pathToFile, pathToDir); + + // Now use the builder to build the same crate independently. + // The files will be reused (we need a place to take a copy from) + RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + + Path pathToZip = tempDir.resolve("%s.eln".formatted(crateName)); + this.saveCrateElnStyle(builtCrate, pathToZip); + + // extract the zip file to a temporary directory + Path extractionPath = tempDir.resolve("extracted_for_testing"); + ensureCrateIsExtractedIn(pathToZip, extractionPath); + HelpFunctions.printFileTree(correctCrate); + HelpFunctions.printFileTree(extractionPath); + + // compare the extracted directory with the correct one + assertTrue(HelpFunctions.compareTwoDir( + correctCrate.toFile(), + extractionPath.toFile())); + HelpFunctions.compareCrateJsonToFileInResources( + builtCrate, + "/json/crate/fileAndDir.json"); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java index 7a469465..0c187029 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java @@ -11,16 +11,16 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderWriterTest extends CrateWriterTest { +class FolderWriterTest implements CommonWriterTest { @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newFolderWriter() .save(crate, target.toAbsolutePath().toString()); } @Override - protected void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { + public void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { FileUtils.copyDirectory(pathToCrate.toFile(), expectedPath.toFile()); } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java new file mode 100644 index 00000000..6cc41a30 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java @@ -0,0 +1,121 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; +import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; +import net.lingala.zip4j.ZipFile; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Assertions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.file.Path; + +/** + * Base Interface for methods required to test all writer strategies. + */ +interface TestableWriterStrategy { + /** + * Saves the crate with the writer fitting to this test class. + * + * @param crate the crate to save + * @param target the target path to the save location + * @throws IOException if an error occurs while saving the crate + */ + void saveCrate(Crate crate, Path target) throws IOException; + + /** + * Ensures the crate is in extracted form in the given path. + * + * @param pathToCrate the path to the crate, may not be a folder yet + * @param expectedPath the path where the crate should be in extracted form + * @throws IOException if an error occurs while extracting the crate + */ + default void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { + try (ZipFile zf = new ZipFile(pathToCrate.toFile())) { + zf.extractAll(expectedPath.toFile().getAbsolutePath()); + } + } + + /** + * Creates a crate structure manually. + * + * @param correctCrate the path to the crate + * @param pathToFile the path to the file + * @param pathToDir the path to the directory + * @throws IOException if an error occurs while creating the crate structure + */ + default void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { + FileUtils.forceMkdir(correctCrate.toFile()); + InputStream fileJson = ZipStreamStrategyTest.class + .getResourceAsStream("/json/crate/fileAndDir.json"); + Assertions.assertNotNull(fileJson); + // fill the directory with expected files and dirs + // starting with the .json of our crate + Path json = correctCrate.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + // create preview + PreviewGenerator.generatePreview(correctCrate.toFile().getAbsolutePath()); + // create the files and directories + FileUtils.writeStringToFile(pathToFile.toFile(), "content of Local File", Charset.defaultCharset()); + // creates the directory and a subdirectory + Path subdir = pathToDir.resolve("subdir"); + FileUtils.forceMkdir(subdir.toFile()); + FileUtils.writeStringToFile( + subdir.resolve("subsubfirst.txt").toFile(), + "content of subsub file in subsubdir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("first.txt").toFile(), + "content of first file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + } + + /** + * Creates a crate resembling the one we manually create in these tests. + * + * @param pathToFile the file to add + * @param pathToSubdir the directory to add + * @return the crate builder + */ + default RoCrate.RoCrateBuilder getCrateWithFileAndDir(Path pathToFile, Path pathToSubdir) { + return new RoCrate.RoCrateBuilder( + "Example RO-Crate", + "The RO-Crate Root Data Entity", + "2024", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(pathToFile) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(pathToSubdir) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index 283b306a..aa3ee5c6 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -8,10 +8,10 @@ /** * @author jejkal */ -class ZipStreamStrategyTest extends CrateWriterTest { +class ZipStreamStrategyTest implements CommonWriterTest { @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { try (FileOutputStream stream = new FileOutputStream(target.toFile())) { Writers.newZipStreamWriter().save(crate, stream); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 3b8b1fc0..80aa91ca 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -5,10 +5,19 @@ import edu.kit.datamanager.ro_crate.Crate; -class ZipWriterTest extends CrateWriterTest { +class ZipWriterTest implements + CommonWriterTest, + ElnFileFormatTest +{ @Override - protected void saveCrate(Crate crate, Path target) throws IOException { + public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipPathWriter() .save(crate, target.toAbsolutePath().toString()); } + + @Override + public void saveCrateElnStyle(Crate crate, Path target) throws IOException { + new CrateWriter<>(new ZipStrategy().usingElnStyle()) + .save(crate, target.toAbsolutePath().toString()); + } } From b46be7f9ce561bbec549ed3b449d9e91b53a9066 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 18:41:00 +0200 Subject: [PATCH 30/46] feat: implement eln style writing for reader.ZipStrategy --- .../ro_crate/preview/CratePreview.java | 75 ++++++++++++++- .../ro_crate/writer/ElnFormatWriter.java | 23 +++++ .../ro_crate/writer/FolderStrategy.java | 17 +++- .../ro_crate/writer/ZipStrategy.java | 96 ++++++++++++++++--- 4 files changed, 196 insertions(+), 15 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index 14ea76ea..46255020 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -2,8 +2,14 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; +import edu.kit.datamanager.ro_crate.writer.FolderStrategy; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import org.apache.commons.io.FileUtils; /** * Interface for the ROCrate preview. This manages the human-readable @@ -15,10 +21,77 @@ */ public interface CratePreview { + /** + * Generate a preview of the crate and store it into the given target directory. + * It is the caller's responsibility to handle, e.g. delete after use, the result + * (The caller takes ownership of the result). + *

+ * IMPORTANT NOTE: This method currently has a default implementation that relies + * on deprecated methods. In future, you will have to implement this method directly. + * + * @param crate the crate to generate a preview for. + * @param targetDir the target directory to store the preview in, + * owned by the caller. + * @throws IOException if an error occurs while generating the preview. + */ + default void generate(Crate crate, File targetDir) throws IOException { + // disable preview generation to avoid recursion, + // as this is usually called in the process of writing a crate + // (including preview) + new CrateWriter<>(new FolderStrategy().disablePreview()) + .save(crate, targetDir.getAbsolutePath()); + this.saveAllToFolder(targetDir); + try (var stream = Files.list(targetDir.toPath())) { + stream + .filter(path -> !path.getFileName().toString().equals("ro-crate-preview.html")) + .filter(path -> !path.getFileName().toString().equals("ro-crate-preview_files")) + .forEach(path -> { + try { + if (Files.isDirectory(path)) { + FileUtils.deleteDirectory(path.toFile()); + } else { + Files.delete(path); + } + } catch (IOException e) { + // Silently ignore deletion errors + } + }); + } + } + + /** + * Takes a crate in form of a zip file and generates a preview of it, + * which will be stored within the crate. + * + * @param zipFile the zip file with the crate, which should receive a preview. + * @throws IOException if an error occurs while saving the preview + * + * @deprecated Use {@link #generate(Crate, File)} instead. + */ + @Deprecated(since = "2.1.0", forRemoval = true) void saveAllToZip(ZipFile zipFile) throws IOException; + /** + * Saves the preview, given by the folder, into the given folder. + * + * @param folder the folder (containing a crate) to save the preview in. + * @throws IOException if an error occurs while saving the preview. + * + * @deprecated Use {@link #generate(Crate, File)} instead. + */ + @Deprecated(since = "2.1.0", forRemoval = true) void saveAllToFolder(File folder) throws IOException; - + + /** + * Saves the preview, given by the metadata, into the given stream. + * + * @param metadata the metadata of the crate to save the preview in. + * @param stream the stream to save the preview in. + * @throws IOException if an error occurs while saving the preview. + * + * @deprecated Use {@link #generate(Crate, File)} instead. + */ + @Deprecated(since = "2.1.0", forRemoval = true) void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java new file mode 100644 index 00000000..a808acef --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -0,0 +1,23 @@ +package edu.kit.datamanager.ro_crate.writer; + +import java.io.IOException; + +public interface ElnFormatWriter extends GenericWriterStrategy { + + /** + * Write in ELN format style, meaning with a root subfolder in the zip file. + * Same as {@link #withRootSubdirectory()}. + * + * @throws IOException if an error occurs + */ + ElnFormatWriter usingElnStyle(); + + /** + * Alias with more generic name for {@link #usingElnStyle()}. + * + * @throws IOException if an error occurs + */ + default ElnFormatWriter withRootSubdirectory() { + return this.usingElnStyle(); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index 8fa5a6bb..96a03de5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -25,6 +25,21 @@ public class FolderStrategy implements GenericWriterStrategy { private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); + protected boolean writePreview = true; + + /** + * For internal use. Skips the preview generation when writing the crate. + * + * @return this instance of FolderStrategy + * + * @deprecated May be removed in future versions. Not intended for public use. + */ + @Deprecated(since = "2.1.0", forRemoval = true) + public FolderStrategy disablePreview() { + this.writePreview = false; + return this; + } + @Override public void save(Crate crate, String destination) throws IOException { File file = new File(destination); @@ -38,7 +53,7 @@ public void save(Crate crate, String destination) throws IOException { FileUtils.copyInputStreamToFile(inputStream, json); inputStream.close(); // save also the preview files to the crate destination - if (crate.getPreview() != null) { + if (crate.getPreview() != null && this.writePreview) { crate.getPreview().saveAllToFolder(file); } for (var e : crate.getUntrackedFiles()) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index 0c8634b1..b6452a9a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -5,42 +5,78 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import edu.kit.datamanager.ro_crate.preview.CratePreview; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.model.ZipParameters; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Matcher; /** * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStrategy implements GenericWriterStrategy { - +public class ZipStrategy implements + GenericWriterStrategy, + ElnFormatWriter +{ private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); + /** + * Defines if the zip file will directly contain the crate, + * or if it will contain a subdirectory with the crate. + */ + protected boolean createRootSubdir = false; + + @Override + public ElnFormatWriter usingElnStyle() { + this.createRootSubdir = true; + return this; + } + @Override public void save(Crate crate, String destination) throws IOException { + String innerFolderName = ""; + if (this.createRootSubdir) { + String dot = Matcher.quoteReplacement("."); + String end = Matcher.quoteReplacement("$"); + innerFolderName = Path.of(destination).getFileName() + .toString() + // remove .zip or .eln from the end of the file name + // (?i) removes case sensitivity + .replaceFirst("(?i)" + dot + "zip" + end, "") + .replaceFirst("(?i)" + dot + "eln" + end, ""); + if (!innerFolderName.endsWith("/")) { + innerFolderName += "/"; + } + } try (ZipFile zipFile = new ZipFile(destination)) { - saveMetadataJson(crate, zipFile); - saveDataEntities(crate, zipFile); + saveMetadataJson(crate, zipFile, innerFolderName); + saveDataEntities(crate, zipFile, innerFolderName); + savePreview(crate, zipFile, innerFolderName); } } - private void saveDataEntities(Crate crate, ZipFile zipFile) throws IOException { + private void saveDataEntities(Crate crate, ZipFile zipFile, String prefix) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - this.saveToZip(dataEntity, zipFile); + this.saveToZip(dataEntity, zipFile, prefix); } } - private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { + private void saveMetadataJson(Crate crate, ZipFile zipFile, String prefix) throws IOException { // write the metadata.json file ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); + zipParameters.setFileNameInZip(prefix + "ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); // we create an JsonNode only to have the file written pretty JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); @@ -49,12 +85,46 @@ private void saveMetadataJson(Crate crate, ZipFile zipFile) throws IOException { // write the ro-crate-metadata zipFile.addStream(inputStream, zipParameters); } - if (crate.getPreview() != null) { - crate.getPreview().saveAllToZip(zipFile); + } + + private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOException { + Optional preview = Optional.ofNullable(crate.getPreview()); + if (preview.isEmpty()) { + return; + } + final String ID = UUID.randomUUID().toString(); + File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + .resolve(ID) + .toFile(); + FileUtils.forceMkdir(tmpPreviewFolder); + FileUtils.forceDeleteOnExit(tmpPreviewFolder); + + preview.get().generate(crate, tmpPreviewFolder); + String[] paths = tmpPreviewFolder.list(); + if (paths == null) { + throw new IOException("No files found in temporary folder"); + } + for (String path : paths) { + File file = tmpPreviewFolder.toPath().resolve(path).toFile(); + if (file.isDirectory()) { + ZipParameters parameters = new ZipParameters(); + parameters.setRootFolderNameInZip(prefix + path); + parameters.setIncludeRootFolder(false); + zipFile.addFolder(file, parameters); + } else { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(prefix + path); + zipFile.addFile(file, zipParameters); + } + } + try { + FileUtils.forceDelete(tmpPreviewFolder); + } catch (IOException e) { + logger.error("Could not delete temporary preview folder: {}", tmpPreviewFolder); } } - private void saveToZip(DataEntity entity, ZipFile zipFile) throws IOException { + private void saveToZip(DataEntity entity, ZipFile zipFile, String prefix) throws IOException { if (entity == null || entity.getPath() == null) { return; } @@ -62,12 +132,12 @@ private void saveToZip(DataEntity entity, ZipFile zipFile) throws IOException { boolean isDirectory = entity.getPath().toFile().isDirectory(); if (isDirectory) { ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(entity.getId()); + parameters.setRootFolderNameInZip(prefix + entity.getId()); parameters.setIncludeRootFolder(false); zipFile.addFolder(entity.getPath().toFile(), parameters); } else { ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(entity.getId()); + zipParameters.setFileNameInZip(prefix + entity.getId()); zipFile.addFile(entity.getPath().toFile(), zipParameters); } } From a296f91aec08c73014981813e31643903ef33f0a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 18:52:10 +0200 Subject: [PATCH 31/46] test: add test for withRootSubdirectory --- .../ro_crate/reader/CommonReaderTest.java | 2 +- .../ro_crate/writer/ZipWriterTest.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java index 6cdc6fa1..1e162a1c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/CommonReaderTest.java @@ -27,7 +27,7 @@ * This parameter is only required to satisfy the generic reader strategy. * @param the type of the reader strategy */ -interface CommonReaderTest< +public interface CommonReaderTest< SOURCE_T, READER_STRATEGY extends GenericReaderStrategy > diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 80aa91ca..d9594b25 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -4,6 +4,13 @@ import java.nio.file.Path; import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.reader.CommonReaderTest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertTrue; class ZipWriterTest implements CommonWriterTest, @@ -20,4 +27,17 @@ public void saveCrateElnStyle(Crate crate, Path target) throws IOException { new CrateWriter<>(new ZipStrategy().usingElnStyle()) .save(crate, target.toAbsolutePath().toString()); } + + @Test + public void testAlias(@TempDir Path tmpDir) throws IOException { + Path zip = tmpDir.resolve("test.eln").toAbsolutePath(); + RoCrate crate = CommonReaderTest.newBaseCrate().build(); + + new CrateWriter<>(new ZipStrategy().withRootSubdirectory()) + .save(crate, zip.toString()); + + assertTrue(zip.toFile().exists(), "The zip file should exist"); + Path extractedPath = tmpDir.resolve("extracted"); + ensureCrateIsExtractedIn(zip, extractedPath); + } } From b25d1eb0c51210d15bfe79b34d93a09a6066b43e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 18:53:43 +0200 Subject: [PATCH 32/46] fix: remove IOException documentation from usingElnStyle and withRootSubdirectory methods --- .../kit/datamanager/ro_crate/writer/ElnFormatWriter.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java index a808acef..40cbf547 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -1,21 +1,15 @@ package edu.kit.datamanager.ro_crate.writer; -import java.io.IOException; - public interface ElnFormatWriter extends GenericWriterStrategy { /** * Write in ELN format style, meaning with a root subfolder in the zip file. * Same as {@link #withRootSubdirectory()}. - * - * @throws IOException if an error occurs */ ElnFormatWriter usingElnStyle(); /** * Alias with more generic name for {@link #usingElnStyle()}. - * - * @throws IOException if an error occurs */ default ElnFormatWriter withRootSubdirectory() { return this.usingElnStyle(); From a09d6f012c3eaf83116048a2c6549bc1d32fdcd7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 9 May 2025 19:08:46 +0200 Subject: [PATCH 33/46] docs: add return documentation for usingElnStyle and withRootSubdirectory methods --- .../edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java index 40cbf547..55f09574 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -5,11 +5,15 @@ public interface ElnFormatWriter extends GenericWriterStrategy usingElnStyle(); /** * Alias with more generic name for {@link #usingElnStyle()}. + * + * @return this writer */ default ElnFormatWriter withRootSubdirectory() { return this.usingElnStyle(); From 00ee66e4035a4df28d9d7257f1eaa9be53c303a2 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 12 May 2025 16:03:26 +0200 Subject: [PATCH 34/46] test: share alias test with all eln writers and implement test for ZipStreamStrategyTest --- ...FormatTest.java => ElnFileWriterTest.java} | 24 ++++++++++++++++++- .../writer/ZipStreamStrategyTest.java | 18 +++++++++++++- .../ro_crate/writer/ZipWriterTest.java | 15 ++++-------- 3 files changed, 44 insertions(+), 13 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{ElnFileFormatTest.java => ElnFileWriterTest.java} (73%) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileWriterTest.java similarity index 73% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileWriterTest.java index feb8dc06..971822e1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileFormatTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ElnFileWriterTest.java @@ -4,6 +4,7 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.reader.CommonReaderTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -12,10 +13,11 @@ import static org.junit.jupiter.api.Assertions.*; -public interface ElnFileFormatTest extends TestableWriterStrategy { +public interface ElnFileWriterTest extends TestableWriterStrategy { /** * Write in ELN format style, meaning with a subfolder in the zip file. + * Must use {@link ElnFormatWriter#usingElnStyle()}. * * @param crate the crate to write * @param target the target path to the save location @@ -23,6 +25,14 @@ public interface ElnFileFormatTest extends TestableWriterStrategy { */ void saveCrateElnStyle(Crate crate, Path target) throws IOException; + /** + * Same as {@link #saveCrateElnStyle(Crate, Path)} but with the alias + * {@link ElnFormatWriter#withRootSubdirectory()}. + * @param crate the crate to write + * @param target the target path to the save location + */ + void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException; + @Test default void testMakesElnStyleCrate(@TempDir Path tempDir) throws IOException { // We need a correct directory to compare with. @@ -67,4 +77,16 @@ default void testMakesElnStyleCrate(@TempDir Path tempDir) throws IOException { builtCrate, "/json/crate/fileAndDir.json"); } + + @Test + default void testAlias(@TempDir Path tmpDir) throws IOException { + Path zip = tmpDir.resolve("test.eln").toAbsolutePath(); + RoCrate crate = CommonReaderTest.newBaseCrate().build(); + + this.saveCrateSubdirectoryStyle(crate, zip); + + assertTrue(zip.toFile().exists(), "The zip file should exist"); + Path extractedPath = tmpDir.resolve("extracted"); + ensureCrateIsExtractedIn(zip, extractedPath); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index aa3ee5c6..ac113667 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -4,11 +4,15 @@ import java.nio.file.Path; import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.RoCrate; /** * @author jejkal */ -class ZipStreamStrategyTest implements CommonWriterTest { +class ZipStreamStrategyTest implements + CommonWriterTest, + ElnFileWriterTest +{ @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -16,4 +20,16 @@ public void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, stream); } } + + @Override + public void saveCrateElnStyle(Crate crate, Path target) throws IOException { + new CrateWriter<>(new ZipStreamStrategy().usingElnStyle()) + .save(crate, target.toAbsolutePath().toString()); + } + + @Override + public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { + new CrateWriter<>(new ZipStreamStrategy().withRootSubdirectory()) + .save(crate, target.toString()); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index d9594b25..3be1b2b3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -14,7 +14,7 @@ class ZipWriterTest implements CommonWriterTest, - ElnFileFormatTest + ElnFileWriterTest { @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -28,16 +28,9 @@ public void saveCrateElnStyle(Crate crate, Path target) throws IOException { .save(crate, target.toAbsolutePath().toString()); } - @Test - public void testAlias(@TempDir Path tmpDir) throws IOException { - Path zip = tmpDir.resolve("test.eln").toAbsolutePath(); - RoCrate crate = CommonReaderTest.newBaseCrate().build(); - + @Override + public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { new CrateWriter<>(new ZipStrategy().withRootSubdirectory()) - .save(crate, zip.toString()); - - assertTrue(zip.toFile().exists(), "The zip file should exist"); - Path extractedPath = tmpDir.resolve("extracted"); - ensureCrateIsExtractedIn(zip, extractedPath); + .save(crate, target.toString()); } } From e2a4bf3f1dc9eac9a1dca92c0a4447b9db39ce11 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 12 May 2025 16:44:41 +0200 Subject: [PATCH 35/46] feat: support ELN format in ZipStreamStrategy --- .../ro_crate/writer/ZipStreamStrategy.java | 114 +++++++++++++++--- .../writer/ZipStreamStrategyTest.java | 4 +- 2 files changed, 99 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index b97b3c2a..12318d24 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -7,16 +7,18 @@ import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Matcher; +import edu.kit.datamanager.ro_crate.preview.CratePreview; import edu.kit.datamanager.ro_crate.util.ZipUtil; -import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,28 +26,72 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStreamStrategy implements GenericWriterStrategy { +public class ZipStreamStrategy implements + GenericWriterStrategy, + ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); + /** + * Defines if the zip file will directly contain the crate, + * or if it will contain a subdirectory with the crate. + */ + protected boolean createRootSubdir = false; + protected String rootSubdirName = "content"; + + @Override + public ElnFormatWriter usingElnStyle() { + this.createRootSubdir = true; + return this; + } + + /** + * Sets the name of a root subdirectory in the zip file. + * Implicitly also enables the creation of a root subdirectory. + * If used for ELN files, note the subdirectory name should be the same as the zip + * files name. + * + * @param name the name of the subdirectory + * @return this instance of ZipStreamStrategy + */ + public ZipStreamStrategy setSubdirectoryName(String name) { + this.rootSubdirName = name; + this.createRootSubdir = true; + return this; + } + @Override public void save(Crate crate, OutputStream destination) throws IOException { + String innerFolderName = ""; + if (this.createRootSubdir) { + String dot = Matcher.quoteReplacement("."); + String end = Matcher.quoteReplacement("$"); + innerFolderName = this.rootSubdirName + // remove .zip or .eln from the end of the file name + // (?i) removes case sensitivity + .replaceFirst("(?i)" + dot + "zip" + end, "") + .replaceFirst("(?i)" + dot + "eln" + end, ""); + if (!innerFolderName.endsWith("/")) { + innerFolderName += "/"; + } + } try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { - saveMetadataJson(crate, zipFile); - saveDataEntities(crate, zipFile); + saveMetadataJson(crate, zipFile, innerFolderName); + saveDataEntities(crate, zipFile, innerFolderName); + savePreview(crate, zipFile, innerFolderName); } } - private void saveDataEntities(Crate crate, ZipOutputStream zipStream) throws IOException { + private void saveDataEntities(Crate crate, ZipOutputStream zipStream, String prefix) throws IOException { for (DataEntity dataEntity : crate.getAllDataEntities()) { - saveToStream(dataEntity, zipStream); + this.saveToStream(dataEntity, zipStream, prefix); } } - private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOException { + private void saveMetadataJson(Crate crate, ZipOutputStream zipStream, String prefix) throws IOException { // write the metadata.json file ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); + zipParameters.setFileNameInZip(prefix + "ro-crate-metadata.json"); ObjectMapper objectMapper = MyObjectMapper.getMapper(); // we create an JsonNode only to have the file written pretty JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); @@ -61,13 +107,47 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) throws IOE } } zipStream.closeEntry(); + } + + private void savePreview(Crate crate, ZipOutputStream zipStream, String prefix) throws IOException { + Optional preview = Optional.ofNullable(crate.getPreview()); + if (preview.isEmpty()) { + return; + } + final String ID = UUID.randomUUID().toString(); + File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + .resolve(ID) + .toFile(); + FileUtils.forceMkdir(tmpPreviewFolder); + FileUtils.forceDeleteOnExit(tmpPreviewFolder); - if (crate.getPreview() != null) { - crate.getPreview().saveAllToStream(str, zipStream); + preview.get().generate(crate, tmpPreviewFolder); + String[] paths = tmpPreviewFolder.list(); + if (paths == null) { + throw new IOException("No files found in temporary folder"); + } + for (String path : paths) { + File file = tmpPreviewFolder.toPath().resolve(path).toFile(); + if (file.isDirectory()) { + ZipUtil.addFolderToZipStream( + zipStream, + file, + prefix + path); + } else { + ZipUtil.addFileToZipStream( + zipStream, + file, + prefix + path); + } + } + try { + FileUtils.forceDelete(tmpPreviewFolder); + } catch (IOException e) { + logger.error("Could not delete temporary preview folder: {}", tmpPreviewFolder); } } - private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws IOException { + private void saveToStream(DataEntity entity, ZipOutputStream zipStream, String prefix) throws IOException { if (entity == null) { return; } @@ -77,12 +157,12 @@ private void saveToStream(DataEntity entity, ZipOutputStream zipStream) throws I ZipUtil.addFolderToZipStream( zipStream, entity.getPath().toAbsolutePath().toString(), - entity.getId()); + prefix + entity.getId()); } else { ZipUtil.addFileToZipStream( zipStream, entity.getPath().toFile(), - entity.getId()); + prefix + entity.getId()); } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index ac113667..c39cebe3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -24,12 +24,12 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { new CrateWriter<>(new ZipStreamStrategy().usingElnStyle()) - .save(crate, target.toAbsolutePath().toString()); + .save(crate, new FileOutputStream(target.toFile())); } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { new CrateWriter<>(new ZipStreamStrategy().withRootSubdirectory()) - .save(crate, target.toString()); + .save(crate, new FileOutputStream(target.toFile())); } } From 8455f4f9f121db8ffbc594762fc2d569528d7778 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 13:56:55 +0200 Subject: [PATCH 36/46] test: add tests for saving previews to ZipOutputStream in PreviewTest --- .../ro_crate/preview/PreviewTest.java | 109 +++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java index 2db987b3..0c1e67b0 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java @@ -1,13 +1,14 @@ package edu.kit.datamanager.ro_crate.preview; +import gg.jte.output.FileOutput; import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -75,6 +76,41 @@ void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); } + @Test + void staticPreviewSaveToZipStream(@TempDir Path dir) throws IOException { + var file1 = dir.resolve("file.html"); + FileUtils.writeStringToFile(file1.toFile(), "random html, does not need to be valid for this test", Charset.defaultCharset()); + + var file2 = dir.resolve("directory"); + var fileInDir = file2.resolve("fileInDir.html"); + FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); + StaticPreview preview = new StaticPreview(file1.toFile(), file2.toFile()); + + ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(dir.resolve("destination.zip").toFile())); + preview.saveAllToStream( + null, // static preview does not need metadata + stream); + stream.flush(); + stream.close(); + + try (ZipFile zf = new ZipFile(dir.resolve("destination.zip").toFile())) { + zf.extractAll(dir.resolve("extracted").toAbsolutePath().toString()); + } + + var e = dir.resolve("extracted"); + var roPreview = e.resolve("ro-crate-preview.html"); + var roDir = e.resolve("ro-crate-preview_files"); + var roDirFile = roDir.resolve("fileInDir.html"); + assertTrue(Files.isRegularFile(roPreview)); + assertTrue(Files.isDirectory(roDir)); + assertTrue(Files.isRegularFile(roDirFile)); + + assertTrue(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), file1.toFile(), String.valueOf(Charset.defaultCharset()))); + assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + + assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + } + @Test void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); @@ -116,6 +152,41 @@ void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } + @Test + void testAutomaticPreviewZipStream(@TempDir Path dir) throws IOException { + AutomaticPreview preview = new AutomaticPreview(); + String metadataPath = "/crates/other/idrc_project/ro-crate-metadata.json"; + Path crate = dir.resolve("crate"); + + File zipFile = dir.resolve("test.zip").toFile(); + { + ZipFile zip = new ZipFile(zipFile); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath); + zip.addStream(crateJson, zipParameters); + crateJson.close(); + } + String metadata = new String( + PreviewTest.class.getResourceAsStream(metadataPath) + .readAllBytes()); + ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile)); + preview.saveAllToStream(metadata, stream); + stream.flush(); + stream.close(); + + try { + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); + preview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid ZIP file for preview."); + } catch (IOException ex) { + //ok + } + new ZipFile(zipFile).extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } + @Test void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { CustomPreview customPreview = new CustomPreview(); @@ -167,4 +238,38 @@ void testCustomPreviewZip(@TempDir Path tmp) throws IOException { assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } + @Test + void testCustomPreviewZipStream(@TempDir Path tmp) throws IOException { + CustomPreview preview = new CustomPreview(); + String metadataPath = "/crates/other/idrc_project/ro-crate-metadata.json"; + Path crate = tmp.resolve("crate"); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + + File zipFile = tmp.resolve("test.zip").toFile(); + { + ZipFile zip = new ZipFile(zipFile); + InputStream crateJson = PreviewTest.class.getResourceAsStream(metadataPath); + zip.addStream(crateJson, zipParameters); + crateJson.close(); + } + String metadata = new String( + PreviewTest.class.getResourceAsStream(metadataPath) + .readAllBytes()); + ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile)); + preview.saveAllToStream(metadata, stream); + stream.flush(); + stream.close(); + + try { + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(tmp.resolve("dddd.zip").toFile()); + preview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid input to preview."); + } catch (IOException ex) { + //ok + } + new ZipFile(zipFile).extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } } From 5ef25d400e71c0799177a97465193a09cc8af3c7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:05:39 +0200 Subject: [PATCH 37/46] fix: enforce correct subfolder name for additional files in StaticPreview.saveAllToStream --- .../edu/kit/datamanager/ro_crate/preview/StaticPreview.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index 5a627e17..cf81aa15 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -66,7 +66,7 @@ public void saveAllToFolder(File folder) throws IOException { public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { ZipUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); if (this.otherFiles != null) { - ZipUtil.addFolderToZipStream(stream, this.otherFiles, this.otherFiles.getName()); + ZipUtil.addFolderToZipStream(stream, this.otherFiles, "ro-crate-preview_files"); } } } From cfa6f73ebaf38aa6f77cdd0a279188b22629124e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:28:40 +0200 Subject: [PATCH 38/46] refactor: prefix reader strategies and writer strategies with Read or Write accordingly. This eases importing reader and writer strategies of the same kind (source/destination type). --- .../ro_crate/preview/CratePreview.java | 4 ++-- .../ro_crate/reader/FolderReader.java | 4 ++-- ...rStrategy.java => ReadFolderStrategy.java} | 2 +- ...{ZipStrategy.java => ReadZipStrategy.java} | 6 +++--- ...rategy.java => ReadZipStreamStrategy.java} | 8 ++++---- .../datamanager/ro_crate/reader/Readers.java | 20 +++++++++---------- .../ro_crate/reader/ZipReader.java | 4 ++-- .../ro_crate/writer/FolderWriter.java | 4 ++-- ...Strategy.java => WriteFolderStrategy.java} | 8 ++++---- ...ZipStrategy.java => WriteZipStrategy.java} | 4 ++-- ...ategy.java => WriteZipStreamStrategy.java} | 8 ++++---- .../datamanager/ro_crate/writer/Writers.java | 6 +++--- .../ro_crate/writer/ZipWriter.java | 4 ++-- .../ro_crate/examples/LearnByExampleTest.java | 8 ++++---- .../ro_crate/reader/FolderReaderTest.java | 8 ++++---- .../ro_crate/reader/ZipReaderTest.java | 10 +++++----- .../ro_crate/reader/ZipStreamReaderTest.java | 10 +++++----- .../writer/ZipStreamStrategyTest.java | 4 ++-- .../ro_crate/writer/ZipWriterTest.java | 8 ++------ 19 files changed, 63 insertions(+), 67 deletions(-) rename src/main/java/edu/kit/datamanager/ro_crate/reader/{FolderStrategy.java => ReadFolderStrategy.java} (92%) rename src/main/java/edu/kit/datamanager/ro_crate/reader/{ZipStrategy.java => ReadZipStrategy.java} (96%) rename src/main/java/edu/kit/datamanager/ro_crate/reader/{ZipStreamStrategy.java => ReadZipStreamStrategy.java} (96%) rename src/main/java/edu/kit/datamanager/ro_crate/writer/{FolderStrategy.java => WriteFolderStrategy.java} (93%) rename src/main/java/edu/kit/datamanager/ro_crate/writer/{ZipStrategy.java => WriteZipStrategy.java} (98%) rename src/main/java/edu/kit/datamanager/ro_crate/writer/{ZipStreamStrategy.java => WriteZipStreamStrategy.java} (96%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index 46255020..a69b47aa 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -6,7 +6,7 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.writer.CrateWriter; -import edu.kit.datamanager.ro_crate.writer.FolderStrategy; +import edu.kit.datamanager.ro_crate.writer.WriteFolderStrategy; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; @@ -38,7 +38,7 @@ default void generate(Crate crate, File targetDir) throws IOException { // disable preview generation to avoid recursion, // as this is usually called in the process of writing a crate // (including preview) - new CrateWriter<>(new FolderStrategy().disablePreview()) + new CrateWriter<>(new WriteFolderStrategy().disablePreview()) .save(crate, targetDir.getAbsolutePath()); this.saveAllToFolder(targetDir); try (var stream = Files.list(targetDir.toPath())) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java index f0436e27..c6268ac1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java @@ -6,7 +6,7 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 * - * @deprecated Use {@link FolderStrategy} instead. + * @deprecated Use {@link ReadFolderStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class FolderReader extends FolderStrategy {} +public class FolderReader extends ReadFolderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadFolderStrategy.java similarity index 92% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ReadFolderStrategy.java index 4b07e01d..927d77de 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadFolderStrategy.java @@ -14,7 +14,7 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -public class FolderStrategy implements GenericReaderStrategy { +public class ReadFolderStrategy implements GenericReaderStrategy { @Override public ObjectNode readMetadataJson(String location) throws IOException { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java index d25a235e..12200616 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java @@ -29,7 +29,7 @@ * persistent location and possibly read it from there, if required. Or use * the ZipWriter to write it back to its source. */ -public class ZipStrategy implements GenericReaderStrategy { +public class ReadZipStrategy implements GenericReaderStrategy { protected final String ID = UUID.randomUUID().toString(); protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipReader/%s/", ID)); @@ -38,7 +38,7 @@ public class ZipStrategy implements GenericReaderStrategy { /** * Crates a ZipReader with the default configuration as described in the class documentation. */ - public ZipStrategy() {} + public ReadZipStrategy() {} /** * Creates a ZipReader which will extract the contents temporary @@ -51,7 +51,7 @@ public ZipStrategy() {} * directory. These subdirectories * will have UUIDs as their names. */ - public ZipStrategy(Path folderPath, boolean shallAddUuidSubfolder) { + public ReadZipStrategy(Path folderPath, boolean shallAddUuidSubfolder) { if (shallAddUuidSubfolder) { this.temporaryFolder = folderPath.resolve(ID); } else { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index f6ed4577..60f908a8 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -25,9 +25,9 @@ * * @author jejkal */ -public class ZipStreamStrategy implements GenericReaderStrategy { +public class ReadZipStreamStrategy implements GenericReaderStrategy { - private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(ReadZipStreamStrategy.class); protected final String ID = UUID.randomUUID().toString(); protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipStreamReader/%s/", ID)); protected boolean isExtracted = false; @@ -36,7 +36,7 @@ public class ZipStreamStrategy implements GenericReaderStrategy { * Crates a ZipStreamReader with the default configuration as described in * the class documentation. */ - public ZipStreamStrategy() { + public ReadZipStreamStrategy() { } /** @@ -49,7 +49,7 @@ public ZipStreamStrategy() { * subdirectories of the given directory. These subdirectories will have * UUIDs as their names. */ - public ZipStreamStrategy(Path folderPath, boolean shallAddUuidSubfolder) { + public ReadZipStreamStrategy(Path folderPath, boolean shallAddUuidSubfolder) { if (shallAddUuidSubfolder) { this.temporaryFolder = folderPath.resolve(ID); } else { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java index 49e09cb1..83a43701 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java @@ -19,10 +19,10 @@ private Readers() {} * * @return A reader configured for ZIP files * - * @see ZipStreamStrategy#ZipStreamStrategy() + * @see ReadZipStreamStrategy#ReadZipStreamStrategy() */ public static CrateReader newZipStreamReader() { - return new CrateReader<>(new ZipStreamStrategy()); + return new CrateReader<>(new ReadZipStreamStrategy()); } /** @@ -33,10 +33,10 @@ public static CrateReader newZipStreamReader() { * @param useUuidSubfolder Whether to create a UUID subfolder under extractPath * @return A reader configured for ZIP files with custom extraction * - * @see ZipStreamStrategy#ZipStreamStrategy(Path, boolean) + * @see ReadZipStreamStrategy#ReadZipStreamStrategy(Path, boolean) */ public static CrateReader newZipStreamReader(Path extractPath, boolean useUuidSubfolder) { - return new CrateReader<>(new ZipStreamStrategy(extractPath, useUuidSubfolder)); + return new CrateReader<>(new ReadZipStreamStrategy(extractPath, useUuidSubfolder)); } /** @@ -44,10 +44,10 @@ public static CrateReader newZipStreamReader(Path extractPath, bool * * @return A reader configured for folders * - * @see FolderStrategy + * @see ReadFolderStrategy */ public static CrateReader newFolderReader() { - return new CrateReader<>(new FolderStrategy()); + return new CrateReader<>(new ReadFolderStrategy()); } /** @@ -55,10 +55,10 @@ public static CrateReader newFolderReader() { * * @return A reader configured for ZIP files * - * @see ZipStrategy#ZipStrategy() + * @see ReadZipStrategy#ReadZipStrategy() */ public static CrateReader newZipPathReader() { - return new CrateReader<>(new ZipStrategy()); + return new CrateReader<>(new ReadZipStrategy()); } /** @@ -69,9 +69,9 @@ public static CrateReader newZipPathReader() { * @param useUuidSubfolder Whether to create a UUID subfolder under extractPath * @return A reader configured for ZIP files with custom extraction * - * @see ZipStrategy#ZipStrategy(Path, boolean) + * @see ReadZipStrategy#ReadZipStrategy(Path, boolean) */ public static CrateReader newZipPathReader(Path extractPath, boolean useUuidSubfolder) { - return new CrateReader<>(new ZipStrategy(extractPath, useUuidSubfolder)); + return new CrateReader<>(new ReadZipStrategy(extractPath, useUuidSubfolder)); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java index ad9bbb01..d92faf94 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java @@ -18,10 +18,10 @@ * persistent location and possibly read it from there, if required. Or use * the ZipWriter to write it back to its source. * - * @deprecated Use {@link ZipStrategy} instead. + * @deprecated Use {@link ReadZipStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class ZipReader extends ZipStrategy { +public class ZipReader extends ReadZipStrategy { /** * Crates a ZipReader with the default configuration as described in the class documentation. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java index 5730104e..1426414e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java @@ -5,7 +5,7 @@ * * @author Nikola Tzotchev on 9.2.2022 г. * - * @deprecated Use {@link FolderStrategy} instead. + * @deprecated Use {@link WriteFolderStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class FolderWriter extends FolderStrategy {} +public class FolderWriter extends WriteFolderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java similarity index 93% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java index 96a03de5..8a4844ed 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java @@ -21,21 +21,21 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -public class FolderStrategy implements GenericWriterStrategy { +public class WriteFolderStrategy implements GenericWriterStrategy { - private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(WriteFolderStrategy.class); protected boolean writePreview = true; /** * For internal use. Skips the preview generation when writing the crate. * - * @return this instance of FolderStrategy + * @return this instance of WriteFolderStrategy * * @deprecated May be removed in future versions. Not intended for public use. */ @Deprecated(since = "2.1.0", forRemoval = true) - public FolderStrategy disablePreview() { + public WriteFolderStrategy disablePreview() { this.writePreview = false; return this; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java similarity index 98% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java index b6452a9a..848f5240 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java @@ -26,11 +26,11 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStrategy implements +public class WriteZipStrategy implements GenericWriterStrategy, ElnFormatWriter { - private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(WriteZipStrategy.class); /** * Defines if the zip file will directly contain the crate, diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index 12318d24..c941bc89 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -26,11 +26,11 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStreamStrategy implements +public class WriteZipStreamStrategy implements GenericWriterStrategy, ElnFormatWriter { - private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(WriteZipStreamStrategy.class); /** * Defines if the zip file will directly contain the crate, @@ -52,9 +52,9 @@ public ElnFormatWriter usingElnStyle() { * files name. * * @param name the name of the subdirectory - * @return this instance of ZipStreamStrategy + * @return this instance of ReadZipStreamStrategy */ - public ZipStreamStrategy setSubdirectoryName(String name) { + public WriteZipStreamStrategy setSubdirectoryName(String name) { this.rootSubdirName = name; this.createRootSubdir = true; return this; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java index 5b691ece..d1a41d66 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java @@ -19,7 +19,7 @@ private Writers() {} * @return a new instance of {@link CrateWriter} for writing to a folder */ public static CrateWriter newFolderWriter() { - return new CrateWriter<>(new FolderStrategy()); + return new CrateWriter<>(new WriteFolderStrategy()); } /** @@ -28,7 +28,7 @@ public static CrateWriter newFolderWriter() { * @return a new instance of {@link CrateWriter} for writing to a zip stream */ public static CrateWriter newZipStreamWriter() { - return new CrateWriter<>(new ZipStreamStrategy()); + return new CrateWriter<>(new WriteZipStreamStrategy()); } /** @@ -37,6 +37,6 @@ public static CrateWriter newZipStreamWriter() { * @return a new instance of {@link CrateWriter} for writing to a zip file */ public static CrateWriter newZipPathWriter() { - return new CrateWriter<>(new ZipStrategy()); + return new CrateWriter<>(new WriteZipStrategy()); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java index f5261003..c9c0735e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java @@ -4,7 +4,7 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. * - * @deprecated Use {@link ZipStrategy} instead. + * @deprecated Use {@link WriteZipStrategy} instead. */ @Deprecated(since = "2.1.0", forRemoval = true) -public class ZipWriter extends ZipStrategy {} +public class ZipWriter extends WriteZipStrategy {} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java index 64733e89..89c5a8f8 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/examples/LearnByExampleTest.java @@ -17,7 +17,7 @@ import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; import edu.kit.datamanager.ro_crate.writer.CrateWriter; -import edu.kit.datamanager.ro_crate.writer.FolderStrategy; +import edu.kit.datamanager.ro_crate.writer.WriteFolderStrategy; import edu.kit.datamanager.ro_crate.writer.GenericWriterStrategy; import edu.kit.datamanager.ro_crate.writer.Writers; import org.apache.commons.io.FileUtils; @@ -336,13 +336,13 @@ void writingAndReadingStrategies(@TempDir Path tempDir) throws IOException { // Now, let's write it to a folder. Note the used strategy could be replaced with your own. Path folder = tempDir.resolve("folderCrate"); - new CrateWriter<>(new FolderStrategy()) + new CrateWriter<>(new WriteFolderStrategy()) .save(crate, folder.toString()); // and read it back. RoCrate read = new CrateReader<>( - // Note: There are two FolderStrategy implementations, one for reading and one for writing. + // Note: There are two WriteFolderStrategy implementations, one for reading and one for writing. // Java is a bit bad with imports, so we use the fully qualified name here. - new edu.kit.datamanager.ro_crate.reader.FolderStrategy() + new edu.kit.datamanager.ro_crate.reader.ReadFolderStrategy() ) .readCrate(folder.toAbsolutePath().toString()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index 8375ba36..83bf4f78 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -17,7 +17,7 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderReaderTest implements CommonReaderTest +class FolderReaderTest implements CommonReaderTest { @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -31,15 +31,15 @@ public Crate readCrate(Path source) throws IOException { } @Override - public FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + public ReadFolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { // This strategy does not support a non-default temporary directory // and will always use the default one. // It also has no state we could make assertions on. - return new FolderStrategy(); + return new ReadFolderStrategy(); } @Override - public Crate readCrate(FolderStrategy strategy, Path source) throws IOException { + public Crate readCrate(ReadFolderStrategy strategy, Path source) throws IOException { return new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 7d71d94e..35b168e1 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -9,8 +9,8 @@ import static org.junit.jupiter.api.Assertions.*; class ZipReaderTest implements - CommonReaderTest, - ElnFileFormatTest + CommonReaderTest, + ElnFileFormatTest { @Override public void saveCrate(Crate crate, Path target) throws IOException { @@ -24,8 +24,8 @@ public Crate readCrate(Path source) throws IOException { } @Override - public ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { - ZipStrategy strategy = new ZipStrategy(tmpDirectory, useUuidSubfolder); + public ReadZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + ReadZipStrategy strategy = new ReadZipStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { assertEquals(strategy.getTemporaryFolder().getFileName().toString(), strategy.getID()); @@ -37,7 +37,7 @@ public ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSu } @Override - public Crate readCrate(ZipStrategy strategy, Path source) throws IOException { + public Crate readCrate(ReadZipStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(source.toAbsolutePath().toString()); assertTrue(strategy.isExtracted()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index a6cca8e6..52feda9d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -11,8 +11,8 @@ class ZipStreamReaderTest implements - CommonReaderTest, - ElnFileFormatTest + CommonReaderTest, + ElnFileFormatTest { /** * At the point of writing this test, @@ -39,8 +39,8 @@ public Crate readCrate(Path source) throws IOException { } @Override - public ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { - ZipStreamStrategy strategy = new ZipStreamStrategy(tmpDirectory, useUuidSubfolder); + public ReadZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + ReadZipStreamStrategy strategy = new ReadZipStreamStrategy(tmpDirectory, useUuidSubfolder); assertFalse(strategy.isExtracted()); if (useUuidSubfolder) { assertEquals(strategy.getTemporaryFolder().getFileName().toString(), strategy.getID()); @@ -52,7 +52,7 @@ public ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean use } @Override - public Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { + public Crate readCrate(ReadZipStreamStrategy strategy, Path source) throws IOException { Crate importedCrate = new CrateReader<>(strategy) .readCrate(new FileInputStream(source.toFile())); assertTrue(strategy.isExtracted()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index c39cebe3..eb9a927f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -23,13 +23,13 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStreamStrategy().usingElnStyle()) + new CrateWriter<>(new WriteZipStreamStrategy().usingElnStyle()) .save(crate, new FileOutputStream(target.toFile())); } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStreamStrategy().withRootSubdirectory()) + new CrateWriter<>(new WriteZipStreamStrategy().withRootSubdirectory()) .save(crate, new FileOutputStream(target.toFile())); } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 3be1b2b3..7e7a6b1d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -5,10 +5,6 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.reader.CommonReaderTest; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -24,13 +20,13 @@ public void saveCrate(Crate crate, Path target) throws IOException { @Override public void saveCrateElnStyle(Crate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStrategy().usingElnStyle()) + new CrateWriter<>(new WriteZipStrategy().usingElnStyle()) .save(crate, target.toAbsolutePath().toString()); } @Override public void saveCrateSubdirectoryStyle(RoCrate crate, Path target) throws IOException { - new CrateWriter<>(new ZipStrategy().withRootSubdirectory()) + new CrateWriter<>(new WriteZipStrategy().withRootSubdirectory()) .save(crate, target.toString()); } } From 92ce1cb69da0f8936a7008bfea4c4dd0ce96cf1f Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:32:16 +0200 Subject: [PATCH 39/46] refactor: remove unused import in ZipWriterTest --- .../java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 7e7a6b1d..bfb29c3d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -6,8 +6,6 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.RoCrate; -import static org.junit.jupiter.api.Assertions.assertTrue; - class ZipWriterTest implements CommonWriterTest, ElnFileWriterTest From 87f790b10a43357fd2218c9ea86477d71e88f498 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 14:40:02 +0200 Subject: [PATCH 40/46] refactor: rename ZipStreamStrategyTest to ZipStreamWriterTest and update references --- .../datamanager/ro_crate/writer/WriteZipStreamStrategy.java | 5 +++++ .../datamanager/ro_crate/writer/TestableWriterStrategy.java | 2 +- .../{ZipStreamStrategyTest.java => ZipStreamWriterTest.java} | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{ZipStreamStrategyTest.java => ZipStreamWriterTest.java} (96%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index c941bc89..16735219 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -37,6 +37,11 @@ public class WriteZipStreamStrategy implements * or if it will contain a subdirectory with the crate. */ protected boolean createRootSubdir = false; + + /** + * In streams, we do not have a file name yet (or do not know it), + * so we need to set a default name for the root subdirectory. + */ protected String rootSubdirName = "content"; @Override diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java index 6cc41a30..079fb766 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/TestableWriterStrategy.java @@ -51,7 +51,7 @@ default void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throw */ default void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { FileUtils.forceMkdir(correctCrate.toFile()); - InputStream fileJson = ZipStreamStrategyTest.class + InputStream fileJson = ZipStreamWriterTest.class .getResourceAsStream("/json/crate/fileAndDir.json"); Assertions.assertNotNull(fileJson); // fill the directory with expected files and dirs diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java similarity index 96% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java index eb9a927f..3f022cf8 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java @@ -9,7 +9,7 @@ /** * @author jejkal */ -class ZipStreamStrategyTest implements +class ZipStreamWriterTest implements CommonWriterTest, ElnFileWriterTest { From b22e70bd837d83edcaab4d56db77fcfea3417db9 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 15:23:03 +0200 Subject: [PATCH 41/46] docs: Improve reader and writer documentation, and related type parameter names --- .../reader/GenericReaderStrategy.java | 8 ++--- .../ro_crate/reader/ReadZipStrategy.java | 22 ++++++++++--- .../reader/ReadZipStreamStrategy.java | 33 +++++++++++++++---- .../ro_crate/writer/CrateWriter.java | 8 ++--- .../ro_crate/writer/ElnFormatWriter.java | 12 +++++-- .../writer/GenericWriterStrategy.java | 6 ++-- 6 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java index 03f6b9c9..6a9f4bd4 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java @@ -8,16 +8,16 @@ * Generic interface for the strategy of the reader class. * This allows for flexible input types when implementing different reading strategies. * - * @param the type of the location parameter + * @param the type which determines the source of the crate */ -public interface GenericReaderStrategy { +public interface GenericReaderStrategy { /** * Read the metadata.json file from the given location. * * @param location the location to read from * @return the parsed metadata.json as ObjectNode */ - ObjectNode readMetadataJson(T location) throws IOException; + ObjectNode readMetadataJson(SOURCE_TYPE location) throws IOException; /** * Read the content from the given location. @@ -25,5 +25,5 @@ public interface GenericReaderStrategy { * @param location the location to read from * @return the content as a File */ - File readContent(T location) throws IOException; + File readContent(SOURCE_TYPE location) throws IOException; } \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java index 12200616..dffbedc7 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java @@ -14,11 +14,20 @@ import java.util.UUID; /** - * A ReaderStrategy implementation which reads from ZipFiles. + * Reads a crate from a ZIP archive (file). *

- * May be used as a dependency for CrateReader. It will unzip - * the ZipFile in a path relative to the directory this application runs in. - * By default, it will be `./.tmp/ro-crate-java/zipReader/$UUID/`. + * This class handles reading and extraction of RO-Crate content from ZIP archives + * into a temporary directory structure on the file system, + * which allows accessing the contained files. + *

+ * Supports ELN-Style crates, + * meaning the crate may be either in the zip archive directly or in a single, + * direct subfolder beneath the root folder (/folder). + *

+ * Note: This implementation checks for up to 50 subdirectories if multiple are present. + * This is to avoid zip bombs, which may contain a lot of subdirectories, + * and at the same time gracefully handle valid crated with hidden subdirectories + * (for example, thumbnails). *

* NOTE: The resulting crate may refer to these temporary files. Therefore, * these files are only being deleted before the JVM exits. If you need to free @@ -36,7 +45,10 @@ public class ReadZipStrategy implements GenericReaderStrategy { protected boolean isExtracted = false; /** - * Crates a ZipReader with the default configuration as described in the class documentation. + * Crates an instance with the default configuration. + *

+ * The default configuration is to extract the ZipFile to + * `./.tmp/ro-crate-java/zipReader/$UUID/`. */ public ReadZipStrategy() {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index 60f908a8..a8213b90 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -19,9 +19,29 @@ import org.slf4j.LoggerFactory; /** - * A ZIP file reader implementation of the StreamReaderStrategy interface. + * Reads a crate from a streamed ZIP archive. + *

* This class handles reading and extraction of RO-Crate content from ZIP archives - * into a temporary directory structure, which allows for accessing the contained files. + * into a temporary directory structure on the file system, + * which allows accessing the contained files. + *

+ * Supports ELN-Style crates, + * meaning the crate may be either in the zip archive directly or in a single, + * direct subfolder beneath the root folder (/folder). + *

+ * Note: This implementation checks for up to 50 subdirectories if multiple are present. + * This is to avoid zip bombs, which may contain a lot of subdirectories, + * and at the same time gracefully handle valid crated with hidden subdirectories + * (for example, thumbnails). + *

+ * NOTE: The resulting crate may refer to these temporary files. Therefore, + * these files are only being deleted before the JVM exits. If you need to free + * space because your application is long-running or creates a lot of + * crates, you may use the getters to retrieve information which will help + * you to clean up manually. Keep in mind that crates may refer to this + * folder after extraction. Use RoCrateWriter to export it so some + * persistent location and possibly read it from there, if required. Or use + * the ZipWriter to write it back to its source. * * @author jejkal */ @@ -33,11 +53,12 @@ public class ReadZipStreamStrategy implements GenericReaderStrategy protected boolean isExtracted = false; /** - * Crates a ZipStreamReader with the default configuration as described in - * the class documentation. + * Crates an instance with the default configuration. + *

+ * The default configuration is to extract the ZipFile to + * `./.tmp/ro-crate-java/zipStreamReader/%UUID/`. */ - public ReadZipStreamStrategy() { - } + public ReadZipStreamStrategy() {} /** * Creates a ZipStreamReader which will extract the contents temporary to diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java index 4ee1be6b..970803ec 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -10,11 +10,11 @@ * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) */ -public class CrateWriter { +public class CrateWriter { - private final GenericWriterStrategy strategy; + private final GenericWriterStrategy strategy; - public CrateWriter(GenericWriterStrategy strategy) { + public CrateWriter(GenericWriterStrategy strategy) { this.strategy = strategy; } @@ -24,7 +24,7 @@ public CrateWriter(GenericWriterStrategy strategy) { * @param crate the crate to write. * @param destination the location where the crate should be written. */ - public void save(Crate crate, DESTINATION destination) throws IOException { + public void save(Crate crate, DESTINATION_TYPE destination) throws IOException { Validator defaultValidation = new Validator(new JsonSchemaValidation()); defaultValidation.validate(crate); this.strategy.save(crate, destination); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java index 55f09574..bd6310cd 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ElnFormatWriter.java @@ -1,6 +1,12 @@ package edu.kit.datamanager.ro_crate.writer; -public interface ElnFormatWriter extends GenericWriterStrategy { +/** + * An Interface for {@link GenericWriterStrategy} implementations which support writing + * ELN-Style crates. + * + * @param the type which determines the destination of the result + */ +public interface ElnFormatWriter extends GenericWriterStrategy { /** * Write in ELN format style, meaning with a root subfolder in the zip file. @@ -8,14 +14,14 @@ public interface ElnFormatWriter extends GenericWriterStrategy usingElnStyle(); + ElnFormatWriter usingElnStyle(); /** * Alias with more generic name for {@link #usingElnStyle()}. * * @return this writer */ - default ElnFormatWriter withRootSubdirectory() { + default ElnFormatWriter withRootSubdirectory() { return this.usingElnStyle(); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java index c0e79779..d06301a0 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java @@ -8,14 +8,14 @@ * Generic interface for the strategy of the writer class. * This allows for flexible output types when implementing different writing strategies. * - * @param the type of the destination parameter + * @param the type of the destination parameter */ -public interface GenericWriterStrategy { +public interface GenericWriterStrategy { /** * Saves the given crate to the specified destination. * * @param crate The crate to save * @param destination The destination where the crate should be saved */ - void save(Crate crate, DESTINATION destination) throws IOException; + void save(Crate crate, DESTINATION_TYPE destination) throws IOException; } From 896018de90f1557a80cb34d972b722313f78d779 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 16:23:32 +0200 Subject: [PATCH 42/46] refactor: started sharing code between WriteZipStrategy and WriteZipStreamStrategy. --- .../ro_crate/preview/AutomaticPreview.java | 12 ++--- .../ro_crate/preview/CustomPreview.java | 8 ++-- .../ro_crate/preview/StaticPreview.java | 6 +-- .../ro_crate/util/FileSystemUtil.java | 48 +++++++++++++++++++ .../util/{ZipUtil.java => ZipStreamUtil.java} | 2 +- .../ro_crate/writer/WriteZipStrategy.java | 23 ++++----- .../writer/WriteZipStreamStrategy.java | 32 ++++++------- 7 files changed, 85 insertions(+), 46 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java rename src/main/java/edu/kit/datamanager/ro_crate/util/{ZipUtil.java => ZipStreamUtil.java} (99%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index 38241476..c403ab1c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -1,6 +1,6 @@ package edu.kit.datamanager.ro_crate.preview; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -10,9 +10,9 @@ import org.apache.commons.io.FileUtils; /** - * The default preview should use the rochtml tool - * (https://www.npmjs.com/package/ro-crate-html-js) for creating a simple - * preview file. + * The default preview should use the + * rochtml tool + * for creating a simple preview file. * * @author Nikola Tzotchev on 6.2.2022 г. * @version 1 @@ -66,13 +66,13 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx if (PreviewGenerator.isRochtmlAvailable()) { try { FileUtils.forceMkdir(new File("temp")); - try (FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json"))) { + try (FileWriter writer = new FileWriter("temp/ro-crate-metadata.json")) { writer.write(metadata); writer.flush(); } if (PreviewGenerator.isRochtmlAvailable()) { PreviewGenerator.generatePreview("temp"); - ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + ZipStreamUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } } finally { try { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index 5c8c9fa2..77300a87 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; @@ -53,7 +53,7 @@ public CustomPreview() { private CustomPreviewModel mapFromJson(String metadata) throws IOException { ObjectMapper mapper = new ObjectMapper(); - JsonNode root = (JsonNode) mapper.readValue(metadata, JsonNode.class); + JsonNode root = mapper.readValue(metadata, JsonNode.class); JsonNode graph = root.get("@graph"); CustomPreviewModel.ROCrate crate = new CustomPreviewModel.ROCrate(); List datasets = new ArrayList<>(); @@ -196,13 +196,13 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx //prepare output folder and writer FileUtils.forceMkdir(new File("temp")); //load and process template - try (FileWriter writer = new FileWriter(new File("temp/ro-crate-preview.html"))) { + try (FileWriter writer = new FileWriter("temp/ro-crate-preview.html")) { //load and process template template.process(dataModel, writer); writer.flush(); } - ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + ZipStreamUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } catch (TemplateException ex) { throw new IOException("Failed to generate preview.", ex); } finally { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index cf81aa15..c3a93629 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -1,6 +1,6 @@ package edu.kit.datamanager.ro_crate.preview; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import java.io.File; import java.io.IOException; @@ -64,9 +64,9 @@ public void saveAllToFolder(File folder) throws IOException { @Override public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { - ZipUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); + ZipStreamUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); if (this.otherFiles != null) { - ZipUtil.addFolderToZipStream(stream, this.otherFiles, "ro-crate-preview_files"); + ZipStreamUtil.addFolderToZipStream(stream, this.otherFiles, "ro-crate-preview_files"); } } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java new file mode 100644 index 00000000..0a1f9e0f --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java @@ -0,0 +1,48 @@ +package edu.kit.datamanager.ro_crate.util; + +import java.util.Collection; +import java.util.regex.Matcher; + +public class FileSystemUtil { + private FileSystemUtil() { + // Utility class, no instantiation + } + + /** + * Removes a specific set of given file extensions from a file name, if present. + * The extensions are case-insensitive. Given "ELN", "eln" or "Eln" will also match. + * The dot (.) before the extension is also assumed and removed implicitly: + *

+ * Example: + * filterExtensionsFromFileName("test.eln", Set.of("ELN")) -> "test" + * + * @param filename the file name to filter + * @param extensionsToRemove the extensions to remove + * @return the filtered file name + */ + public static String filterExtensionsFromFileName(String filename, Collection extensionsToRemove) { + String dot = Matcher.quoteReplacement("."); + String end = Matcher.quoteReplacement("$"); + for (String extension : extensionsToRemove) { + // (?i) removes case sensitivity + filename = filename.replaceFirst("(?i)" + dot + extension + end, ""); + } + return filename; + } + + /** + * Ensures that a given path ends with a trailing slash. + * + * @param path the path to check + * @return the path with a trailing slash if it didn't have one, or the original path + */ + public static String ensureTrailingSlash(String path) { + if (path == null || path.isEmpty()) { + return path; + } + if (!path.endsWith("/")) { + return path + "/"; + } + return path; + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipStreamUtil.java similarity index 99% rename from src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java rename to src/main/java/edu/kit/datamanager/ro_crate/util/ZipStreamUtil.java index c1da0137..b8eb2ef1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipStreamUtil.java @@ -11,7 +11,7 @@ * * @author jejkal */ -public class ZipUtil { +public class ZipStreamUtil { /** * Adds a folder and its contents to a ZipOutputStream. diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java index 848f5240..04812054 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java @@ -6,6 +6,7 @@ import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import edu.kit.datamanager.ro_crate.preview.CratePreview; +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -19,8 +20,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Optional; +import java.util.Set; import java.util.UUID; -import java.util.regex.Matcher; /** * Implementation of the writing strategy to provide a way of writing crates to @@ -31,6 +32,7 @@ public class WriteZipStrategy implements ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(WriteZipStrategy.class); + public static final String TMP_DIR = "./.tmp/ro-crate-java/writer-zip-strategy/"; /** * Defines if the zip file will directly contain the crate, @@ -48,17 +50,10 @@ public ElnFormatWriter usingElnStyle() { public void save(Crate crate, String destination) throws IOException { String innerFolderName = ""; if (this.createRootSubdir) { - String dot = Matcher.quoteReplacement("."); - String end = Matcher.quoteReplacement("$"); - innerFolderName = Path.of(destination).getFileName() - .toString() - // remove .zip or .eln from the end of the file name - // (?i) removes case sensitivity - .replaceFirst("(?i)" + dot + "zip" + end, "") - .replaceFirst("(?i)" + dot + "eln" + end, ""); - if (!innerFolderName.endsWith("/")) { - innerFolderName += "/"; - } + innerFolderName = FileSystemUtil.filterExtensionsFromFileName( + Path.of(destination).getFileName().toString(), + Set.of("ELN", "ZIP")); + innerFolderName = FileSystemUtil.ensureTrailingSlash(innerFolderName); } try (ZipFile zipFile = new ZipFile(destination)) { saveMetadataJson(crate, zipFile, innerFolderName); @@ -93,7 +88,7 @@ private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOE return; } final String ID = UUID.randomUUID().toString(); - File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + File tmpPreviewFolder = Path.of(TMP_DIR) .resolve(ID) .toFile(); FileUtils.forceMkdir(tmpPreviewFolder); @@ -102,7 +97,7 @@ private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOE preview.get().generate(crate, tmpPreviewFolder); String[] paths = tmpPreviewFolder.list(); if (paths == null) { - throw new IOException("No files found in temporary folder"); + throw new IOException("No preview files found in temporary folder. Preview generation failed."); } for (String path : paths) { File file = tmpPreviewFolder.toPath().resolve(path).toFile(); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index 16735219..8c54cc95 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -11,11 +11,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Optional; +import java.util.Set; import java.util.UUID; -import java.util.regex.Matcher; import edu.kit.datamanager.ro_crate.preview.CratePreview; -import edu.kit.datamanager.ro_crate.util.ZipUtil; +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; +import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -31,6 +32,7 @@ public class WriteZipStreamStrategy implements ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(WriteZipStreamStrategy.class); + public static final String TMP_DIR = "./.tmp/ro-crate-java/writer-zip-stream-strategy/"; /** * Defines if the zip file will directly contain the crate, @@ -69,16 +71,10 @@ public WriteZipStreamStrategy setSubdirectoryName(String name) { public void save(Crate crate, OutputStream destination) throws IOException { String innerFolderName = ""; if (this.createRootSubdir) { - String dot = Matcher.quoteReplacement("."); - String end = Matcher.quoteReplacement("$"); - innerFolderName = this.rootSubdirName - // remove .zip or .eln from the end of the file name - // (?i) removes case sensitivity - .replaceFirst("(?i)" + dot + "zip" + end, "") - .replaceFirst("(?i)" + dot + "eln" + end, ""); - if (!innerFolderName.endsWith("/")) { - innerFolderName += "/"; - } + innerFolderName = FileSystemUtil.filterExtensionsFromFileName( + this.rootSubdirName, + Set.of("ELN", "ZIP")); + innerFolderName = FileSystemUtil.ensureTrailingSlash(innerFolderName); } try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { saveMetadataJson(crate, zipFile, innerFolderName); @@ -120,7 +116,7 @@ private void savePreview(Crate crate, ZipOutputStream zipStream, String prefix) return; } final String ID = UUID.randomUUID().toString(); - File tmpPreviewFolder = Path.of("./.tmp/ro-crate-java/writer-zipStrategy/") + File tmpPreviewFolder = Path.of(TMP_DIR) .resolve(ID) .toFile(); FileUtils.forceMkdir(tmpPreviewFolder); @@ -129,17 +125,17 @@ private void savePreview(Crate crate, ZipOutputStream zipStream, String prefix) preview.get().generate(crate, tmpPreviewFolder); String[] paths = tmpPreviewFolder.list(); if (paths == null) { - throw new IOException("No files found in temporary folder"); + throw new IOException("No preview files found in temporary folder. Preview generation failed."); } for (String path : paths) { File file = tmpPreviewFolder.toPath().resolve(path).toFile(); if (file.isDirectory()) { - ZipUtil.addFolderToZipStream( + ZipStreamUtil.addFolderToZipStream( zipStream, file, prefix + path); } else { - ZipUtil.addFileToZipStream( + ZipStreamUtil.addFileToZipStream( zipStream, file, prefix + path); @@ -159,12 +155,12 @@ private void saveToStream(DataEntity entity, ZipOutputStream zipStream, String p boolean isDirectory = entity.getPath().toFile().isDirectory(); if (isDirectory) { - ZipUtil.addFolderToZipStream( + ZipStreamUtil.addFolderToZipStream( zipStream, entity.getPath().toAbsolutePath().toString(), prefix + entity.getId()); } else { - ZipUtil.addFileToZipStream( + ZipStreamUtil.addFileToZipStream( zipStream, entity.getPath().toFile(), prefix + entity.getId()); From fd9b84db617f66b276e5b365724759da412b9416 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:11:48 +0200 Subject: [PATCH 43/46] refactor: factor out common tmp directory handling in zip readers --- .../ro_crate/reader/ReadZipStrategy.java | 8 ++--- .../reader/ReadZipStreamStrategy.java | 13 ++------ .../ro_crate/util/FileSystemUtil.java | 24 ++++++++++++++ .../ro_crate/util/FileSystemUtilTest.java | 31 +++++++++++++++++++ 4 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java index dffbedc7..5a2474b5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStrategy.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -94,12 +95,7 @@ public boolean isExtracted() { private void readCrate(String location) throws IOException { File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } + FileSystemUtil.mkdirOrDeleteContent(folder); // extract try (ZipFile zf = new ZipFile(location)) { zf.extractAll(temporaryFolder.toAbsolutePath().toString()); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java index a8213b90..afaa217e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReadZipStreamStrategy.java @@ -11,6 +11,8 @@ import java.io.OutputStream; import java.nio.file.Path; import java.util.UUID; + +import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import net.lingala.zip4j.io.inputstream.ZipInputStream; import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; @@ -108,16 +110,7 @@ public boolean isExtracted() { */ private void readCrate(InputStream stream) throws IOException { File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.exists()) { - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - } else { - FileUtils.forceMkdir(folder); - } + FileSystemUtil.mkdirOrDeleteContent(folder); LocalFileHeader localFileHeader; int readLen; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java index 0a1f9e0f..a7e0ce2e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/FileSystemUtil.java @@ -1,6 +1,11 @@ package edu.kit.datamanager.ro_crate.util; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; import java.util.Collection; +import java.util.Objects; import java.util.regex.Matcher; public class FileSystemUtil { @@ -45,4 +50,23 @@ public static String ensureTrailingSlash(String path) { } return path; } + + /** + * Creates a directory or deletes its content if it already exists. + * + * @param folder the folder to create or delete content from + * @throws IOException if an I/O error occurs + */ + public static void mkdirOrDeleteContent(File folder) throws IOException { + boolean isNonEmptyDir = folder.exists() + && folder.isDirectory() + && Objects.requireNonNull(folder.listFiles()).length > 0; + boolean isFile = folder.exists() + && !folder.isDirectory(); + + if (isNonEmptyDir || isFile) { + FileUtils.forceDelete(folder); + } + FileUtils.forceMkdir(folder); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java b/src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java new file mode 100644 index 00000000..fac64527 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/util/FileSystemUtilTest.java @@ -0,0 +1,31 @@ +package edu.kit.datamanager.ro_crate.util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.*; + +class FileSystemUtilTest { + + @ValueSource(strings = { + "test", + "test/", + "test/test", + "test/test/", + "test/test/test", + "test/test/test/" + }) + @ParameterizedTest + void ensureTrailingSlash(String value) { + String result = FileSystemUtil.ensureTrailingSlash(value); + assertTrue(result.endsWith("/"), "The result should end with a trailing slash."); + } + + @SuppressWarnings("ConstantValue") + @Test + void ensureTrailingSlashNull() { + String result = FileSystemUtil.ensureTrailingSlash(null); + assertNull(result, "The result should be null."); + } +} \ No newline at end of file From 1e2da1171b275f9c0521f7df3a795f0b6fc486c2 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:13:31 +0200 Subject: [PATCH 44/46] refactor: simplify WriteZipStrategy by delegating save operations to WriteZipStreamStrategy --- .../ro_crate/writer/WriteZipStrategy.java | 118 +----------------- 1 file changed, 6 insertions(+), 112 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java index 04812054..003f7973 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStrategy.java @@ -1,27 +1,13 @@ package edu.kit.datamanager.ro_crate.writer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; -import edu.kit.datamanager.ro_crate.preview.CratePreview; -import edu.kit.datamanager.ro_crate.util.FileSystemUtil; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.model.ZipParameters; -import org.apache.commons.io.FileUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; +import java.io.OutputStream; /** * Implementation of the writing strategy to provide a way of writing crates to @@ -32,108 +18,16 @@ public class WriteZipStrategy implements ElnFormatWriter { private static final Logger logger = LoggerFactory.getLogger(WriteZipStrategy.class); - public static final String TMP_DIR = "./.tmp/ro-crate-java/writer-zip-strategy/"; - - /** - * Defines if the zip file will directly contain the crate, - * or if it will contain a subdirectory with the crate. - */ - protected boolean createRootSubdir = false; + protected ElnFormatWriter delegate = new WriteZipStreamStrategy(); @Override public ElnFormatWriter usingElnStyle() { - this.createRootSubdir = true; + this.delegate = this.delegate.withRootSubdirectory(); return this; } @Override public void save(Crate crate, String destination) throws IOException { - String innerFolderName = ""; - if (this.createRootSubdir) { - innerFolderName = FileSystemUtil.filterExtensionsFromFileName( - Path.of(destination).getFileName().toString(), - Set.of("ELN", "ZIP")); - innerFolderName = FileSystemUtil.ensureTrailingSlash(innerFolderName); - } - try (ZipFile zipFile = new ZipFile(destination)) { - saveMetadataJson(crate, zipFile, innerFolderName); - saveDataEntities(crate, zipFile, innerFolderName); - savePreview(crate, zipFile, innerFolderName); - } - } - - private void saveDataEntities(Crate crate, ZipFile zipFile, String prefix) throws IOException { - for (DataEntity dataEntity : crate.getAllDataEntities()) { - this.saveToZip(dataEntity, zipFile, prefix); - } - } - - private void saveMetadataJson(Crate crate, ZipFile zipFile, String prefix) throws IOException { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(prefix + "ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - } - } - - private void savePreview(Crate crate, ZipFile zipFile, String prefix) throws IOException { - Optional preview = Optional.ofNullable(crate.getPreview()); - if (preview.isEmpty()) { - return; - } - final String ID = UUID.randomUUID().toString(); - File tmpPreviewFolder = Path.of(TMP_DIR) - .resolve(ID) - .toFile(); - FileUtils.forceMkdir(tmpPreviewFolder); - FileUtils.forceDeleteOnExit(tmpPreviewFolder); - - preview.get().generate(crate, tmpPreviewFolder); - String[] paths = tmpPreviewFolder.list(); - if (paths == null) { - throw new IOException("No preview files found in temporary folder. Preview generation failed."); - } - for (String path : paths) { - File file = tmpPreviewFolder.toPath().resolve(path).toFile(); - if (file.isDirectory()) { - ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(prefix + path); - parameters.setIncludeRootFolder(false); - zipFile.addFolder(file, parameters); - } else { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(prefix + path); - zipFile.addFile(file, zipParameters); - } - } - try { - FileUtils.forceDelete(tmpPreviewFolder); - } catch (IOException e) { - logger.error("Could not delete temporary preview folder: {}", tmpPreviewFolder); - } - } - - private void saveToZip(DataEntity entity, ZipFile zipFile, String prefix) throws IOException { - if (entity == null || entity.getPath() == null) { - return; - } - - boolean isDirectory = entity.getPath().toFile().isDirectory(); - if (isDirectory) { - ZipParameters parameters = new ZipParameters(); - parameters.setRootFolderNameInZip(prefix + entity.getId()); - parameters.setIncludeRootFolder(false); - zipFile.addFolder(entity.getPath().toFile(), parameters); - } else { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(prefix + entity.getId()); - zipFile.addFile(entity.getPath().toFile(), zipParameters); - } + this.delegate.save(crate, new FileOutputStream(destination)); } } From 9cf26b157dbd72f79830c5556189cae23fb84df7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:25:13 +0200 Subject: [PATCH 45/46] docs: add IOException documentation to readCrate method and clarify CrateWriter type parameter --- .../java/edu/kit/datamanager/ro_crate/reader/CrateReader.java | 2 ++ .../java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 7dcadbdc..5132e44b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -83,6 +83,8 @@ public CrateReader(GenericReaderStrategy strategy) { * * @param location the location of the ro-crate to be read * @return the read RO-crate + * + * @throws IOException if the crate cannot be read */ public RoCrate readCrate(T location) throws IOException { // get the ro-crate-metadata.json diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java index 970803ec..caba67f9 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -9,6 +9,8 @@ /** * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) + * + * @param the type which determines the destination of the result */ public class CrateWriter { From 06b2146eab9698794ec1263fb104449c1a72605a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 19 May 2025 17:40:38 +0200 Subject: [PATCH 46/46] test: add not-null-assertions for crate and person entity --- .../ro_crate/crate/realexamples/RealTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java index ae206b7c..19ba2a83 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java @@ -12,7 +12,6 @@ import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.externalproviders.personprovider.OrcidProvider; -import edu.kit.datamanager.ro_crate.reader.CrateReader; import edu.kit.datamanager.ro_crate.reader.Readers; import org.apache.commons.io.FileUtils; @@ -23,17 +22,17 @@ import java.nio.charset.Charset; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -class RealTest { - - @SuppressWarnings("java:S2699") // disable warning about missing assertions +class RealTest +{ @Test void testWithIDRCProject(@TempDir Path temp) throws IOException { - - CrateReader reader = Readers.newFolderReader(); final String locationMetadataFile = "/crates/other/idrc_project/ro-crate-metadata.json"; - Crate crate = reader.readCrate(RealTest.class.getResource("/crates/other/idrc_project").getPath()); + Crate crate = Readers.newFolderReader() + .readCrate(RealTest.class.getResource("/crates/other/idrc_project").getPath()); + assertNotNull(crate); HelpFunctions.compareCrateJsonToFileInResources(crate, locationMetadataFile); Path newFile = temp.resolve("new_file.txt"); @@ -47,6 +46,7 @@ void testWithIDRCProject(@TempDir Path temp) throws IOException { .build()); PersonEntity person = OrcidProvider.getPerson("https://orcid.org/0000-0001-9842-9718"); + assertNotNull(person); crate.addContextualEntity(person); // problem