From d3ed8c91f982464f6d9a43a2912398091ec20754 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 14 Mar 2025 17:13:46 +0100 Subject: [PATCH 01/88] fix: revert back to surpressing pull_request triggers and append debug output --- .github/workflows/gradle.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 38f6634f..18d4f548 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -18,9 +18,8 @@ env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} jobs: - # avoid double CI runs on push and PR, from https://github.com/orgs/community/discussions/57827 build: - if: github.event_name != 'push' || github.event.push.head.repo.full_name != github.event.push.base.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] @@ -28,6 +27,11 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Debug trigger conditions + run: | + echo "github.event_name=${{ github.event_name }}" + echo "github.event.pull_request.head.repo.full_name=${{ github.event.pull_request.head.repo.full_name }}" + echo "github.event.pull_request.base.repo.full_name=${{ github.event.pull_request.base.repo.full_name }}" - uses: actions/checkout@v4 - name: Set up openJDK version uses: actions/setup-java@v4 From 8029da8fffea0556b0cd23e80bfd28a89d655e9c Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 14 Mar 2025 17:40:58 +0100 Subject: [PATCH 02/88] ci: debug information and remove trigger limitation for now --- .github/workflows/gradle.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 18d4f548..22acca1e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -19,7 +19,6 @@ env: jobs: build: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] @@ -32,6 +31,8 @@ jobs: echo "github.event_name=${{ github.event_name }}" echo "github.event.pull_request.head.repo.full_name=${{ github.event.pull_request.head.repo.full_name }}" echo "github.event.pull_request.base.repo.full_name=${{ github.event.pull_request.base.repo.full_name }}" + echo "github.event.action=${{ github.event.action }}" + echo "github.event=${{ toJson(github.event) }}" - uses: actions/checkout@v4 - name: Set up openJDK version uses: actions/setup-java@v4 From 3450e3874498de271cd0a20039004162f052a8a0 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 14 Mar 2025 17:42:54 +0100 Subject: [PATCH 03/88] ci: remove full event information as it breaks something --- .github/workflows/gradle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 22acca1e..9a20850b 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -32,7 +32,6 @@ jobs: echo "github.event.pull_request.head.repo.full_name=${{ github.event.pull_request.head.repo.full_name }}" echo "github.event.pull_request.base.repo.full_name=${{ github.event.pull_request.base.repo.full_name }}" echo "github.event.action=${{ github.event.action }}" - echo "github.event=${{ toJson(github.event) }}" - uses: actions/checkout@v4 - name: Set up openJDK version uses: actions/setup-java@v4 From 49fe1da014ee5ec87ec07ff32d35b89be2ee3c9b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 23:11:45 +0000 Subject: [PATCH 04/88] chore(deps): update plugin io.freefair.maven-publish-java to v8.13.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2c87b3b1..be59eb88 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ plugins { // Publishing of JAR to Nexus instances (e.g., OSSRH) // https://github.com/gradle-nexus/publish-plugin id "io.github.gradle-nexus.publish-plugin" version "2.0.0" - id "io.freefair.maven-publish-java" version "8.13" + id "io.freefair.maven-publish-java" version "8.13.1" } group 'edu.kit.datamanager' From a92b19f8284aea56a29c2b189a81e5dd35ff88a4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 28 Mar 2025 23:41:29 +0000 Subject: [PATCH 05/88] chore(deps): update dependency com.github.fslev:json-compare to v7 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2c87b3b1..4e0ca5d4 100644 --- a/build.gradle +++ b/build.gradle @@ -53,7 +53,7 @@ dependencies { // read from and write to zip files implementation group: 'net.lingala.zip4j', name: 'zip4j', version: '2.11.5' // compare json documents in tests - implementation 'com.github.fslev:json-compare:6.18' + implementation 'com.github.fslev:json-compare:7.0' // url validator implementation group: 'commons-validator', name: 'commons-validator', version: '1.9.0' // logging From 79c65b4f87f1b51c69a2a2d5e0f19523fd994019 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Tue, 1 Apr 2025 15:45:08 +0200 Subject: [PATCH 06/88] Added zip stream writer, minor cleanup --- build.gradle | 5 ++ gradle.properties | 8 +- .../ro_crate/entities/data/DataEntity.java | 69 +++++++++----- .../ro_crate/preview/AutomaticPreview.java | 39 ++++---- .../ro_crate/preview/CratePreview.java | 2 +- .../ro_crate/preview/CustomPreview.java | 90 +++++++++---------- .../ro_crate/writer/StreamWriterStrategy.java | 14 +++ .../ro_crate/writer/ZipStreamWriter.java | 81 +++++++++++++++++ .../ro_crate/writer/ZipWriter.java | 3 + 9 files changed, 225 insertions(+), 86 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java diff --git a/build.gradle b/build.gradle index 2c87b3b1..0fb61995 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,9 @@ println "Building ${name} version: ${version}" println "JDK version: ${JavaVersion.current()}" println "Profile (system property): ${System.getProperty('profile')}" +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + if (JavaVersion.current() == JavaVersion.VERSION_17) { println "Setting encoding to UTF-8 manually" compileJava.options.encoding = "UTF-8" @@ -65,6 +68,8 @@ dependencies { implementation 'org.glassfish:jakarta.json:2.0.1' } +logging.captureStandardOutput LogLevel.INFO + def signingTasks = tasks.withType(Sign) tasks.withType(AbstractPublishToMaven).configureEach{ mustRunAfter(signingTasks) diff --git a/gradle.properties b/gradle.properties index fb7cb539..f9c7a26f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1,7 @@ -version=2.0.2 \ No newline at end of file +version=2.0.2 +action.custom-1=install +action.custom-1.args=--configure-on-demand -w -x check clean publishToMavenLocal +action.custom-2=jacoco +action.custom-2.args=--configure-on-demand -w clean check jacocoTestReport +action.custom-3=releaseNewVersion +action.custom-3.args=--configure-on-demand -w -x check -Prelease release \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 28293a54..9f8b6292 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -7,13 +7,16 @@ import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -44,9 +47,9 @@ public DataEntity(AbstractDataEntityBuilder entityBuilder) { /** * Adds an author ID to the entity. - * + * * Calling this multiple times will add multiple author IDs. - * + * * @param id the identifier of the author. */ public void addAuthorId(String id) { @@ -69,6 +72,31 @@ public void saveToZip(ZipFile zipFile) throws ZipException { } } + /** + * If the data entity contains a physical file. This method will write it + * when the crate is being written to a zip archive. + * + * @param zipStream The zip output stream where it should be written. + * @throws ZipException when something goes wrong with the writing to the + * zip file. + * @throws IOException If opening the file input stream fails. + */ + public void saveToStream(ZipOutputStream zipStream) throws ZipException, IOException { + if (this.path != null) { + byte[] buff = new byte[4096]; + int readLen; + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(this.getId()); + zipStream.putNextEntry(zipParameters); + try (InputStream inputStream = new FileInputStream(this.path.toFile())) { + while ((readLen = inputStream.read(buff)) != -1) { + zipStream.write(buff, 0, readLen); + } + } + zipStream.closeEntry(); + } + } + /** * If the data entity contains a physical file. This method will write it * when the crate is being written to a folder. @@ -99,14 +127,14 @@ abstract static class AbstractDataEntityBuilder { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index 7124d203..3bcc3f4d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -6,32 +6,33 @@ import org.apache.commons.io.FileUtils; /** - * The default preview should use the rochtml tool (https://www.npmjs.com/package/ro-crate-html-js) - * for creating a simple preview file. + * The default preview should use the rochtml tool + * (https://www.npmjs.com/package/ro-crate-html-js) for creating a simple + * preview file. * * @author Nikola Tzotchev on 6.2.2022 г. * @version 1 */ public class AutomaticPreview implements CratePreview { - public AutomaticPreview() { - } + public AutomaticPreview() { + } - @Override - public void saveAllToZip(ZipFile zipFile) { - try { - // extract the .json file so we can run the "rochtml" tool on it" - zipFile.extractFile("ro-crate-metadata.json", "temp"); - PreviewGenerator.generatePreview("temp"); - zipFile.addFile("temp/ro-crate-preview.html"); - FileUtils.deleteDirectory(new File("temp")); - } catch (IOException e) { - e.printStackTrace(); + @Override + public void saveAllToZip(ZipFile zipFile) { + try { + // extract the .json file so we can run the "rochtml" tool on it" + zipFile.extractFile("ro-crate-metadata.json", "temp"); + PreviewGenerator.generatePreview("temp"); + zipFile.addFile("temp/ro-crate-preview.html"); + FileUtils.deleteDirectory(new File("temp")); + } catch (IOException e) { + e.printStackTrace(); + } } - } - @Override - public void saveAllToFolder(File folder) { - PreviewGenerator.generatePreview(folder.getAbsolutePath()); - } + @Override + public void saveAllToFolder(File folder) { + PreviewGenerator.generatePreview(folder.getAbsolutePath()); + } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index 0a225b8e..b828f5f8 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -15,5 +15,5 @@ public interface CratePreview { void saveAllToZip(ZipFile zipFile); void saveAllToFolder(File folder); - + } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index 4e3bc3f4..0139c031 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -8,61 +8,61 @@ import org.apache.commons.io.FileUtils; /** - * This class represents the custom preview of a crate, - * which means html files created from outside sources. + * This class represents the custom preview of a crate, which means html files + * created from outside sources. * * @author Nikola Tzotchev on 12.2.2022 г. * @version 1 */ public class CustomPreview implements CratePreview { - private final File metadataHtml; - private final File otherFiles; + private final File metadataHtml; + private final File otherFiles; - public CustomPreview(File metadataHtml, File otherFiles) { - this.metadataHtml = metadataHtml; - this.otherFiles = otherFiles; - } - - public CustomPreview(File metadataHtml) { - this.metadataHtml = metadataHtml; - this.otherFiles = null; - } + public CustomPreview(File metadataHtml, File otherFiles) { + this.metadataHtml = metadataHtml; + this.otherFiles = otherFiles; + } - @Override - public void saveAllToZip(ZipFile zipFile) { - if (this.metadataHtml != null) { - try { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-preview.html"); - zipFile.addFile(this.metadataHtml, zipParameters); - } catch (ZipException e) { - System.err.println("Exception writing preview html to zip"); - } + public CustomPreview(File metadataHtml) { + this.metadataHtml = metadataHtml; + this.otherFiles = null; } - if (this.otherFiles != null) { - try { - zipFile.addFolder(this.otherFiles); - zipFile.renameFile(this.otherFiles.getName() + "/", "ro-crate-preview_files/"); - } catch (ZipException e) { - System.err.println("Exception writing preview files to zip"); - } + + @Override + public void saveAllToZip(ZipFile zipFile) { + if (this.metadataHtml != null) { + try { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-preview.html"); + zipFile.addFile(this.metadataHtml, zipParameters); + } catch (ZipException e) { + System.err.println("Exception writing preview html to zip"); + } + } + if (this.otherFiles != null) { + try { + zipFile.addFolder(this.otherFiles); + zipFile.renameFile(this.otherFiles.getName() + "/", "ro-crate-preview_files/"); + } catch (ZipException e) { + System.err.println("Exception writing preview files to zip"); + } + } } - } - @Override - public void saveAllToFolder(File folder) { - try { - if (this.metadataHtml != null) { - File fileInCrate = folder.toPath().resolve("ro-crate-preview.html").toFile(); - FileUtils.copyFile(this.metadataHtml, fileInCrate); - } - if (this.otherFiles != null) { - File folderName = folder.toPath().resolve("ro-crate-preview_files").toFile(); - FileUtils.copyDirectory(this.otherFiles, folderName); - } - } catch (IOException e) { - e.printStackTrace(); + @Override + public void saveAllToFolder(File folder) { + try { + if (this.metadataHtml != null) { + File fileInCrate = folder.toPath().resolve("ro-crate-preview.html").toFile(); + FileUtils.copyFile(this.metadataHtml, fileInCrate); + } + if (this.otherFiles != null) { + File folderName = folder.toPath().resolve("ro-crate-preview_files").toFile(); + FileUtils.copyDirectory(this.otherFiles, folderName); + } + } catch (IOException e) { + e.printStackTrace(); + } } - } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java new file mode 100644 index 00000000..4762e126 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java @@ -0,0 +1,14 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import java.io.OutputStream; + +/** + * Strategy for writing of crates to streams. + * + * @author jejkal + */ +public interface StreamWriterStrategy { + + void save(Crate crate, OutputStream destination); +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java new file mode 100644 index 00000000..7dd4e3a0 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java @@ -0,0 +1,81 @@ +package edu.kit.datamanager.ro_crate.writer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import net.lingala.zip4j.model.ZipParameters; + +/** + * Implementation of the writing strategy to provide a way of writing crates to + * a zip archive. + */ +public class ZipStreamWriter implements StreamWriterStrategy { + + @Override + public void save(Crate crate, OutputStream destination) { + + try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { + saveMetadataJson(crate, zipFile); + saveDataEntities(crate, zipFile); + } catch (IOException e) { + // can not close ZipOutputStream (threw Exception) + } + } + + private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { + for (DataEntity dataEntity : crate.getAllDataEntities()) { + try { + dataEntity.saveToStream(zipStream); + } catch (IOException e) { + System.out.println("could not save " + dataEntity.getId() + " to zip stream!"); + } + } + } + + private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { + try { + // write the metadata.json file + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + // write the ro-crate-metadata + + byte[] buff = new byte[4096]; + int readLen; + zipStream.putNextEntry(zipParameters); + try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { + while ((readLen = inputStream.read(buff)) != -1) { + zipStream.write(buff, 0, readLen); + } + } + zipStream.closeEntry(); + + //TODO: Preview not written as creation requires extracted crate, + //which is not possible if directly written to stream. To be done later. + /* if (crate.getPreview() != null) { + crate.getPreview().saveAllToStream(zipStream); + }*/ + } catch (ZipException | JsonProcessingException e) { + System.out.println("Exception writing ro-crate-metadata.json file to zip"); + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java index bcc73f45..a0654d2b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java @@ -14,6 +14,7 @@ import java.nio.charset.StandardCharsets; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; /** @@ -24,6 +25,8 @@ public class ZipWriter implements WriterStrategy { @Override public void save(Crate crate, String destination) { + + try (ZipFile zipFile = new ZipFile(destination)) { saveMetadataJson(crate, zipFile); saveDataEntities(crate, zipFile); From 8d95e20be74a5b336f910a9f65d68f542d56f7f4 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Thu, 3 Apr 2025 16:02:13 +0200 Subject: [PATCH 07/88] Added tests for ZipStreamWriter, started adding integrated preview (tests not working, yet) --- .gitignore | 3 + build.gradle | 3 + .../ro_crate/entities/data/DataEntity.java | 1 + .../ro_crate/entities/data/DataSetEntity.java | 185 +++++++++++------- .../ro_crate/preview/AutomaticPreview.java | 18 +- .../preview/DefaultPreviewGenerator.java | 173 ++++++++++++++++ .../ro_crate/preview/PreviewGenerator.java | 112 ++++++----- .../ro_crate/writer/FolderWriter.java | 70 +++---- .../ro_crate/writer/RoCrateWriter.java | 56 ++++-- .../ro_crate/writer/StreamWriterStrategy.java | 24 ++- .../ro_crate/writer/ZipStreamWriter.java | 15 +- .../ro_crate/writer/ZipWriter.java | 85 ++++---- .../resources/templates/default_preview.jte | 54 +++++ .../ro_crate/writer/ZipStreamWriterTest.java | 171 ++++++++++++++++ .../ro_crate/writer/ZipWriterTest.java | 5 + 15 files changed, 757 insertions(+), 218 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java create mode 100644 src/main/resources/templates/default_preview.jte create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java diff --git a/.gitignore b/.gitignore index 68eac014..2e6ac719 100644 --- a/.gitignore +++ b/.gitignore @@ -169,4 +169,7 @@ gradle-app.setting # gradle/wrapper/gradle-wrapper.properties ### Gradle Patch ### +.DS_Store **/build/ +**/jte-classes +**/temp diff --git a/build.gradle b/build.gradle index 0fb61995..cefc1805 100644 --- a/build.gradle +++ b/build.gradle @@ -66,6 +66,9 @@ dependencies { // metadata validation, profiles based on JSON schema implementation group: "com.networknt", name: "json-schema-validator", version: "1.5.6" implementation 'org.glassfish:jakarta.json:2.0.1' + //JTE for template processing + implementation('gg.jte:jte:3.1.16') + } logging.captureStandardOutput LogLevel.INFO diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 9f8b6292..1b2377a3 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -68,6 +68,7 @@ public void saveToZip(ZipFile zipFile) throws ZipException { if (this.path != null) { ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip(this.getId()); + System.out.println("ADD FILE " + this.path.toFile()); zipFile.addFile(this.path.toFile(), zipParameters); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index c9f08d3f..ff3ef0bc 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -4,11 +4,17 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import edu.kit.datamanager.ro_crate.entities.serializers.HasPartSerializer; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.HashSet; import java.util.Set; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import net.lingala.zip4j.model.ZipParameters; /** * A helping class for the creating of Data entities of type Dataset. @@ -18,90 +24,133 @@ */ public class DataSetEntity extends DataEntity { - public static final String TYPE = "Dataset"; - - @JsonSerialize(using = HasPartSerializer.class) - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public Set hasPart; - - /** - * Constructor for instantiating a Dataset entity from the builder. - * - * @param entityBuilder the builder passed as argument. - */ - public DataSetEntity(AbstractDataSetBuilder entityBuilder) { - super(entityBuilder); - this.hasPart = entityBuilder.hasPart; - this.addType(TYPE); - } - - public void removeFromHasPart(String str) { - this.hasPart.remove(str); - } - - @Override - public void saveToZip(ZipFile zipFile) throws ZipException { - if (this.getPath() != null) { - zipFile.addFolder(this.getPath().toFile()); + public static final String TYPE = "Dataset"; + + @JsonSerialize(using = HasPartSerializer.class) + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public Set hasPart; + + /** + * Constructor for instantiating a Dataset entity from the builder. + * + * @param entityBuilder the builder passed as argument. + */ + public DataSetEntity(AbstractDataSetBuilder entityBuilder) { + super(entityBuilder); + this.hasPart = entityBuilder.hasPart; + this.addType(TYPE); } - } - public void addToHasPart(String id) { - this.hasPart.add(id); - } + public void removeFromHasPart(String str) { + this.hasPart.remove(str); + } + + @Override + public void saveToZip(ZipFile zipFile) throws ZipException { + if (this.getPath() != null) { + zipFile.addFolder(this.getPath().toFile()); + } + } - public boolean hasInHasPart(String id) { - return this.hasPart.contains(id); - } + @Override + public void saveToStream(ZipOutputStream zipOutputStream) throws ZipException, IOException { + if (this.getPath() != null) { + addFolderToZip(zipOutputStream, this.getPath().toAbsolutePath().toString(), this.getPath().getFileName().toString()); + } + } - abstract static class AbstractDataSetBuilder> extends - AbstractDataEntityBuilder { + public void addFolderToZip(ZipOutputStream zipOutputStream, String folderPath, String parentPath) throws IOException { + File folder = new File(folderPath); + if (!folder.exists() || !folder.isDirectory()) { + throw new IllegalArgumentException("The provided folder path is not a valid directory: " + folderPath); + } + + File[] files = folder.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + String zipEntryPath = parentPath.isEmpty() ? file.getName() : parentPath + "/" + file.getName(); + if (file.isDirectory()) { + addFolderToZip(zipOutputStream, file.getAbsolutePath(), zipEntryPath); + } else { + addFileToZip(zipOutputStream, file, zipEntryPath); + } + } + } - Set hasPart; + private void addFileToZip(ZipOutputStream zipOutputStream, File file, String zipEntryPath) throws IOException { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(zipEntryPath); + zipOutputStream.putNextEntry(zipParameters); + try (InputStream inputStream = new FileInputStream(file)) { + byte[] buffer = new byte[4096]; + int len; + while ((len = inputStream.read(buffer)) != -1) { + zipOutputStream.write(buffer, 0, len); + } + } - public AbstractDataSetBuilder() { - this.hasPart = new HashSet<>(); + zipOutputStream.closeEntry(); } - public T setHasPart(Set hastPart) { - this.hasPart = hastPart; - return self(); + public void addToHasPart(String id) { + this.hasPart.add(id); } - public T addToHasPart(DataEntity dataEntity) { - if (dataEntity != null) { - this.hasPart.add(dataEntity.getId()); - this.relatedItems.add(dataEntity.getId()); - } - return self(); + public boolean hasInHasPart(String id) { + return this.hasPart.contains(id); } - public T addToHasPart(String dataEntity) { - if (dataEntity != null) { - this.hasPart.add(dataEntity); - this.relatedItems.add(dataEntity); - } - return self(); + abstract static class AbstractDataSetBuilder> extends + AbstractDataEntityBuilder { + + Set hasPart; + + public AbstractDataSetBuilder() { + this.hasPart = new HashSet<>(); + } + + public T setHasPart(Set hastPart) { + this.hasPart = hastPart; + return self(); + } + + public T addToHasPart(DataEntity dataEntity) { + if (dataEntity != null) { + this.hasPart.add(dataEntity.getId()); + this.relatedItems.add(dataEntity.getId()); + } + return self(); + } + + public T addToHasPart(String dataEntity) { + if (dataEntity != null) { + this.hasPart.add(dataEntity); + this.relatedItems.add(dataEntity); + } + return self(); + } + + @Override + public abstract DataSetEntity build(); } - @Override - public abstract DataSetEntity build(); - } - - /** - * Builder for the helping class DataSetEntity. - */ - public static final class DataSetBuilder extends AbstractDataSetBuilder { + /** + * Builder for the helping class DataSetEntity. + */ + public static final class DataSetBuilder extends AbstractDataSetBuilder { - @Override - public DataSetBuilder self() { - return this; - } + @Override + public DataSetBuilder self() { + return this; + } - @Override - public DataSetEntity build() { - return new DataSetEntity(this); + @Override + public DataSetEntity build() { + return new DataSetEntity(this); + } } - } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index 3bcc3f4d..96617e13 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -23,7 +23,11 @@ public void saveAllToZip(ZipFile zipFile) { try { // extract the .json file so we can run the "rochtml" tool on it" zipFile.extractFile("ro-crate-metadata.json", "temp"); - PreviewGenerator.generatePreview("temp"); + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview("temp"); + } else { + DefaultPreviewGenerator.generatePreview(FileUtils.readFileToString(new File("temp/ro-crate-metadata.json"), "UTF-8"), zipFile); + } zipFile.addFile("temp/ro-crate-preview.html"); FileUtils.deleteDirectory(new File("temp")); } catch (IOException e) { @@ -33,6 +37,16 @@ public void saveAllToZip(ZipFile zipFile) { @Override public void saveAllToFolder(File folder) { - PreviewGenerator.generatePreview(folder.getAbsolutePath()); + + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview(folder.getAbsolutePath()); + } else { + try { + DefaultPreviewGenerator.generatePreview(FileUtils.readFileToString(new File(folder, "ro-crate-metadata.json"), "UTF-8"), folder); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java new file mode 100644 index 00000000..581057d5 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java @@ -0,0 +1,173 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.kit.datamanager.ro_crate.preview; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import gg.jte.CodeResolver; +import gg.jte.ContentType; +import gg.jte.TemplateEngine; +import gg.jte.TemplateOutput; +import gg.jte.output.StringOutput; +import gg.jte.resolve.ResourceCodeResolver; +import java.io.ByteArrayInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.model.ZipParameters; + +/** + * + * @author jejkal + */ +public class DefaultPreviewGenerator { + + public static class ROCrateModel { + + public ROCrate crate; + public List datasets; + public List files; + } + + public static class ROCrate { + + public String name; + public String description; + public String type; + public String license; + public String datePublished; + public List hasPart; + } + + public static class Part { + + public String id; + public String name; + } + + public static class Dataset { + + public String id; + public String name; + public String description; + } + + public static class File { + + public String id; + public String name; + public String description; + public String contentSize; + public String encodingFormat; + } + + public static ROCrateModel mapFromJson(String metadata) throws IOException { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode root = objectMapper.readTree(metadata); + JsonNode graph = root.get("@graph"); + + ROCrate crate = new ROCrate(); + List datasets = new ArrayList<>(); + List files = new ArrayList<>(); + + if (graph.isArray()) { + for (JsonNode node : graph) { + String id = node.get("@id").asText(); + String type = node.get("@type").asText(); + + if ("Dataset".equals(type) && "./".equals(id)) { + crate.name = node.get("name").asText(); + crate.description = node.get("description").asText(); + crate.type = type; + crate.license = node.get("license").get("@id").asText(); + crate.datePublished = node.get("datePublished").asText(); + crate.hasPart = new ArrayList<>(); + + if (node.has("hasPart")) { + for (JsonNode part : node.get("hasPart")) { + Part p = new Part(); + p.id = part.get("@id").asText(); + p.name = "Unknown"; // Name will be set later + crate.hasPart.add(p); + } + } + } else if ("Dataset".equals(type)) { + Dataset dataset = new Dataset(); + dataset.id = id; + dataset.name = node.get("name").asText(); + dataset.description = node.get("description").asText(); + datasets.add(dataset); + } else if ("File".equals(type)) { + File file = new File(); + file.id = id; + file.name = node.get("name").asText(); + file.description = node.get("description").asText(); + file.contentSize = node.get("contentSize").asText(); + file.encodingFormat = node.get("encodingFormat").asText(); + files.add(file); + } + } + } + + // Update Part names using dataset and file lists + for (Part part : crate.hasPart) { + for (Dataset dataset : datasets) { + if (dataset.id.equals(part.id)) { + part.name = dataset.name; + } + } + for (File file : files) { + if (file.id.equals(part.id)) { + part.name = file.name; + } + } + } + + ROCrateModel model = new ROCrateModel(); + model.crate = crate; + model.datasets = datasets; + model.files = files; + return model; + } + + public static void generatePreview(String metadata, ZipFile zip) throws IOException { + ROCrateModel model = mapFromJson(metadata); + + Map input = new HashMap(); + input.put("model", model); + + CodeResolver codeResolver = new ResourceCodeResolver("templates"); // This is the directory where your .jte files are located. + TemplateEngine templateEngine = TemplateEngine.create(codeResolver, ContentType.Html); + TemplateOutput output = new StringOutput(); + templateEngine.render("default_preview.jte", input, output); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-preview.html"); + zip.addStream(new ByteArrayInputStream(output.toString().getBytes()), zipParameters); + } + + public static void generatePreview(String metadata, java.io.File folder) throws IOException { + ROCrateModel model = mapFromJson(metadata); + + Map input = new HashMap(); + input.put("model", model); + + CodeResolver codeResolver = new ResourceCodeResolver("templates"); // This is the directory where your .jte files are located. + TemplateEngine templateEngine = TemplateEngine.create(codeResolver, ContentType.Html); + + TemplateOutput output = new StringOutput(); + templateEngine.render("default_preview.jte", input, output); + System.out.println(output.toString()); + FileWriter w = new FileWriter(new java.io.File("ro-crate-preview.html")); + w.write(output.toString()); + w.flush(); + w.close(); + + } + +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java index 55339ab4..b348c565 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java @@ -1,57 +1,81 @@ package edu.kit.datamanager.ro_crate.preview; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStreamReader; /** - * Class responsible for the generation of the human-readable representation of the metadata. - * For this to work the nodejs library rochtml has to be installed. - * This can be done using: - * npm install -g ro-crate-html-js - * or - * npm install ro-crate-html-js + * Class responsible for the generation of the human-readable representation of + * the metadata. For this to work the nodejs library rochtml has to be + * installed. This can be done using: npm install -g ro-crate-html-js or npm + * install ro-crate-html-js */ public class PreviewGenerator { - private static final String command = "rochtml"; - - /** - * The method that from the location of the crate generates the html file. - * - * @param location the location of the crate in the filesystem. - */ - public static void generatePreview(String location) { - ProcessBuilder builder = new ProcessBuilder(); - // this is the equivalent of "rochtml dir/ro-crate-metadata.json" - // check if we are running on windows or unix - String os = System.getProperty("os.name").toLowerCase(); - if (os.contains("win")) { - builder.command("cmd.exe", "/c", command + " " + location + "/ro-crate-metadata.json"); - - } else { - builder.command("sh", "-c", command + " " + location + "/ro-crate-metadata.json"); + private static final String command = "rochtml"; + + public static boolean isRochtmlAvailable() { + ProcessBuilder builder = new ProcessBuilder(); + // this is the equivalent of "rochtml dir/ro-crate-metadata.json" + // check if we are running on windows or unix + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) { + builder.command("cmd.exe", "/c", command); + + } else { + builder.command("sh", "-c", command); + } + + Process process; + try { + process = builder.start(); + + int exitVal = process.waitFor(); + System.out.println("EXIT " + exitVal); + return exitVal == 0; + } catch (InterruptedException | IOException ex) { + + } + return false; } - Process process; - try { - process = builder.start(); - StringBuilder output = new StringBuilder(); - - BufferedReader reader = new BufferedReader( - new InputStreamReader(process.getInputStream())); - - String line; - while ((line = reader.readLine()) != null) { - output.append(line).append("\n"); - } - reader.close(); - int exitVal = process.waitFor(); - if (exitVal != 0) { - //abnormal... - throw new Exception("failed command"); - } - } catch (Exception e) { - e.printStackTrace(); + /** + * The method that from the location of the crate generates the html file. + * + * @param location the location of the crate in the filesystem. + */ + public static void generatePreview(String location) { + ProcessBuilder builder = new ProcessBuilder(); + // this is the equivalent of "rochtml dir/ro-crate-metadata.json" + // check if we are running on windows or unix + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) { + builder.command("cmd.exe", "/c", command + " " + location + "/ro-crate-metadata.json"); + + } else { + builder.command("sh", "-c", command + " " + location + "/ro-crate-metadata.json"); + } + + Process process; + try { + process = builder.start(); + StringBuilder output = new StringBuilder(); + + BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream())); + + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append("\n"); + } + reader.close(); + int exitVal = process.waitFor(); + if (exitVal != 0) { + //abnormal... + throw new Exception("failed command"); + } + } catch (Exception e) { + e.printStackTrace(); + } } - } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java index 0be2a96d..5be66701 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java @@ -13,6 +13,8 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A class for writing a crate to a folder. @@ -22,41 +24,41 @@ */ public class FolderWriter implements WriterStrategy { - @Override - public void save(Crate crate, String destination) { - File file = new File(destination); - try { - FileUtils.forceMkdir(file); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + private static Logger logger = LoggerFactory.getLogger(FolderWriter.class); - File json = new File(destination, "ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(inputStream, json); - inputStream.close(); - // save also the preview files to the crate destination - if (crate.getPreview() != null) { - crate.getPreview().saveAllToFolder(file); - } - for (var e : crate.getUntrackedFiles()) { - if (e.isDirectory()) { - FileUtils.copyDirectoryToDirectory(e, file); - } else { - FileUtils.copyFileToDirectory(e, file); + @Override + public void save(Crate crate, String destination) { + File file = new File(destination); + try { + FileUtils.forceMkdir(file); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + + File json = new File(destination, "ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(inputStream, json); + inputStream.close(); + // save also the preview files to the crate destination + if (crate.getPreview() != null) { + crate.getPreview().saveAllToFolder(file); + } + for (var e : crate.getUntrackedFiles()) { + if (e.isDirectory()) { + FileUtils.copyDirectoryToDirectory(e, file); + } else { + FileUtils.copyFileToDirectory(e, file); + } + } + } catch (IOException e) { + logger.error("Error creating destination directory!", e); + } + for (DataEntity dataEntity : crate.getAllDataEntities()) { + try { + dataEntity.savetoFile(file); + } catch (IOException e) { + logger.error("Cannot save " + dataEntity.getId() + " to destination folder!", e); + } } - } - } catch (IOException e) { - System.out.println("Error creating destination directory!"); - e.printStackTrace(); - } - for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.savetoFile(file); - } catch (IOException e) { - System.out.println("Cannot save " + dataEntity.getId() + " to destination folder!"); - e.printStackTrace(); - } } - } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java index 09b28c5a..de3c35ef 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java @@ -3,29 +3,49 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; +import java.io.OutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * The class used for writing (exporting) crates. - * The class uses a strategy pattern for writing crates as different formats. - * (zip, folders, etc.) + * The class used for writing (exporting) crates. The class uses a strategy + * pattern for writing crates as different formats. (zip, folders, etc.) */ public class RoCrateWriter { - private final WriterStrategy writer; + private static Logger logger = LoggerFactory.getLogger(FolderWriter.class); - public RoCrateWriter(WriterStrategy writer) { - this.writer = writer; - } + private final WriterStrategy writer; - /** - * This method saves the crate to a destination provided. - * - * @param crate the crate to write. - * @param destination the location where the crate should be written. - */ - public void save(Crate crate, String destination) { - Validator defaultValidation = new Validator(new JsonSchemaValidation()); - defaultValidation.validate(crate); - this.writer.save(crate, destination); - } + public RoCrateWriter(WriterStrategy writer) { + this.writer = writer; + } + + /** + * This method saves the crate to a destination provided. + * + * @param crate the crate to write. + * @param destination the location where the crate should be written. + */ + public void save(Crate crate, String destination) { + Validator defaultValidation = new Validator(new JsonSchemaValidation()); + defaultValidation.validate(crate); + this.writer.save(crate, destination); + } + + /** + * This method saves the crate to a destination provided. + * + * @param crate the crate to write. + * @param destination the location where the crate should be written. + */ + public void save(Crate crate, OutputStream destination) { + Validator defaultValidation = new Validator(new JsonSchemaValidation()); + defaultValidation.validate(crate); + if (writer instanceof StreamWriterStrategy) { + ((StreamWriterStrategy) this.writer).save(crate, destination); + } else { + logger.error("Provided writer does not implement StreamWriterStrategy. Please use 'save(Crate crate, String destination)'."); + } + } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java index 4762e126..04deb659 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java @@ -1,14 +1,36 @@ package edu.kit.datamanager.ro_crate.writer; import edu.kit.datamanager.ro_crate.Crate; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.OutputStream; +import org.slf4j.LoggerFactory; /** * Strategy for writing of crates to streams. * * @author jejkal */ -public interface StreamWriterStrategy { +public interface StreamWriterStrategy extends WriterStrategy { + + static org.slf4j.Logger logger = LoggerFactory.getLogger(StreamWriterStrategy.class); + + /** + * Default override of save interface from WriterStrategy. The override + * assumes, that destination is a file, which is used as output stream. If + * this assumption is not true, this call will fail. + * + * @param crate The crate to write. + * @param destination The destination, which is supposed to be a file. + */ + default void save(Crate crate, String destination) { + try { + save(crate, new FileOutputStream(new File(destination))); + } catch (FileNotFoundException ex) { + logger.error("Failed save crate to destination " + destination, ex); + } + } void save(Crate crate, OutputStream destination); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java index 7dd4e3a0..a3b9e72a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java @@ -1,6 +1,5 @@ package edu.kit.datamanager.ro_crate.writer; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -9,14 +8,14 @@ import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import java.io.ByteArrayInputStream; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Implementation of the writing strategy to provide a way of writing crates to @@ -24,6 +23,8 @@ */ public class ZipStreamWriter implements StreamWriterStrategy { + private static Logger logger = LoggerFactory.getLogger(ZipStreamWriter.class); + @Override public void save(Crate crate, OutputStream destination) { @@ -32,6 +33,7 @@ public void save(Crate crate, OutputStream destination) { saveDataEntities(crate, zipFile); } catch (IOException e) { // can not close ZipOutputStream (threw Exception) + logger.error("Failed to save ro-crate to zip stream.", e); } } @@ -40,7 +42,7 @@ private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { try { dataEntity.saveToStream(zipStream); } catch (IOException e) { - System.out.println("could not save " + dataEntity.getId() + " to zip stream!"); + logger.error("Could not save " + dataEntity.getId() + " to zip stream!", e); } } } @@ -71,11 +73,8 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { /* if (crate.getPreview() != null) { crate.getPreview().saveAllToStream(zipStream); }*/ - } catch (ZipException | JsonProcessingException e) { - System.out.println("Exception writing ro-crate-metadata.json file to zip"); - e.printStackTrace(); } catch (IOException e) { - e.printStackTrace(); + logger.error("Exception writing ro-crate-metadata.json file to zip.", e); } } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java index a0654d2b..67d8024a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java @@ -14,58 +14,57 @@ import java.nio.charset.StandardCharsets; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Implementation of the writing strategy - * to provide a way of writing crates to a zip archive. + * Implementation of the writing strategy to provide a way of writing crates to + * a zip archive. */ public class ZipWriter implements WriterStrategy { - @Override - public void save(Crate crate, String destination) { - - - try (ZipFile zipFile = new ZipFile(destination)) { - saveMetadataJson(crate, zipFile); - saveDataEntities(crate, zipFile); - } catch (IOException e) { - // can not close ZipFile (threw Exception) + private static Logger logger = LoggerFactory.getLogger(ZipWriter.class); + + @Override + public void save(Crate crate, String destination) { + try (ZipFile zipFile = new ZipFile(destination)) { + saveMetadataJson(crate, zipFile); + saveDataEntities(crate, zipFile); + } catch (IOException e) { + // can not close ZipFile (threw Exception) + logger.error("Failed to write ro-crate to destination " + destination + ".", e); + } } - } - private void saveDataEntities(Crate crate, ZipFile zipFile) { - for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.saveToZip(zipFile); - } catch (ZipException e) { - System.out.println("could not save " + dataEntity.getId() + " to zip file!"); - } + private void saveDataEntities(Crate crate, ZipFile zipFile) { + for (DataEntity dataEntity : crate.getAllDataEntities()) { + try { + dataEntity.saveToZip(zipFile); + } catch (ZipException e) { + logger.error("Could not save " + dataEntity.getId() + " to zip file!", e); + } + } } - } - private void saveMetadataJson(Crate crate, ZipFile zipFile) { - try { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - inputStream.close(); - if (crate.getPreview() != null) { - crate.getPreview().saveAllToZip(zipFile); - } - } catch (ZipException | JsonProcessingException e) { - System.out.println("Exception writing ro-crate-metadata.json file to zip"); - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); + private void saveMetadataJson(Crate crate, ZipFile zipFile) { + try { + // write the metadata.json file + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + // write the ro-crate-metadata + zipFile.addStream(inputStream, zipParameters); + inputStream.close(); + if (crate.getPreview() != null) { + crate.getPreview().saveAllToZip(zipFile); + } + } catch (IOException e) { + logger.error("Exception writing ro-crate-metadata.json file to zip.", e); + } } - } } diff --git a/src/main/resources/templates/default_preview.jte b/src/main/resources/templates/default_preview.jte new file mode 100644 index 00000000..85ffd2b8 --- /dev/null +++ b/src/main/resources/templates/default_preview.jte @@ -0,0 +1,54 @@ +@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.ROCrateModel +@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.Part +@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.Dataset +@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.File + +@param ROCrateModel model + + + + + + + RO-Crate Viewer + + + +
+

RO-Crate Metadata

+
+

${model.crate.name}

+

Description: ${model.crate.description}

+

Type: ${model.crate.type}

+

License: License

+

Date Published: ${model.crate.datePublished}

+

Contains:

+
    + @for(Part part : model.crate.hasPart) { +
  • ${part.name}
    • + @endfor} +
+
+ + @for(Dataset dataset : model.datasets) { +
+

Dataset: ${dataset.name}

+

Description: ${dataset.description}

+
+ @endfor} + + @if(!model.files.isEmpty()){ + @for(File file : model.files) { +
+

File: ${file.name}

+

Description: ${file.description}

+

Size: ${file.contentSize} bytes

+

Format: ${file.encodingFormat}

+
+ @endfor} + }@else

No files.

+ @endif +
+ + + diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java new file mode 100644 index 00000000..b34002a7 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java @@ -0,0 +1,171 @@ +package edu.kit.datamanager.ro_crate.writer; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.file.Path; + +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; +import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; +import net.lingala.zip4j.ZipFile; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * @author jejkal + */ +class ZipStreamWriterTest { + + @Test + void testWritingToZipStream(@TempDir Path tempDir) throws IOException { + // create the RO_crate directory in the tempDir + Path roDir = tempDir.resolve("ro_dir"); + FileUtils.forceMkdir(roDir.toFile()); + + // the .json of our crate + InputStream fileJson= + ZipStreamWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + + // fill the expected directory with files and dirs + + Path json = roDir.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + + PreviewGenerator.generatePreview(roDir.toString()); + + Path file1 = roDir.resolve("cp7glop.ai"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = roDir.resolve("dir"); + FileUtils.forceMkdir(dirInCrate.toFile()); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(file1) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()) + .build(); + + // safe the crate in the test.zip file + Path test = tempDir.resolve("test.zip"); + // create a Writer for writing RoCrates to zip + RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipStreamWriter()); + + + // save the content of the roCrate to the dest zip + roCrateZipWriter.save(roCrate, test.toString()); + Path res = tempDir.resolve("dest"); + try (ZipFile zf = new ZipFile(test.toFile())) { + zf.extractAll(res.toString()); + } + assertTrue(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); + + // just so we know the metadata is still valid + HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + } + + + @Test + void testWritingToZipFail(@TempDir Path tempDir) throws IOException { + // create the RO_crate directory in the tempDir + Path roDir = tempDir.resolve("ro_dir"); + FileUtils.forceMkdir(roDir.toFile()); + + // the .json of our crate + InputStream fileJson= + ZipStreamWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + + // fill the expected directory with files and dirs + + Path json = roDir.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + + PreviewGenerator.generatePreview(roDir.toFile().getAbsolutePath()); + + Path file1 = roDir.resolve("input.txt"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = roDir.resolve("dir"); + FileUtils.forceMkdir(dirInCrate.toFile()); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + // false file, this test case should fal + Path falseFile = tempDir.resolve("new"); + FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(falseFile) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .build(); + + // safe the crate in the test.zip file + Path test = tempDir.resolve("test.zip"); + // create a Writer for writing RoCrates to zip + RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipStreamWriter()); + // save the content of the roCrate to the dest zip + roCrateZipWriter.save(roCrate, test.toFile().getAbsolutePath()); + Path res = tempDir.resolve("dest"); + try (ZipFile zf = new ZipFile(test.toFile())) { + zf.extractAll(res.toFile().getAbsolutePath()); + } + assertFalse(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); + + // just so we know the metadata is still valid + HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 641de7af..910e1b1b 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -14,6 +14,7 @@ import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; +import java.nio.file.Paths; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; @@ -86,6 +87,10 @@ void testWritingToZip(@TempDir Path tempDir) throws IOException { RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, test.toString()); + roCrateZipWriter.save(roCrate, "/Users/jejkal/cra.zip"); + + + Path res = tempDir.resolve("dest"); try (ZipFile zf = new ZipFile(test.toFile())) { zf.extractAll(res.toString()); From ad72cc4c3a49f1ddc0df2a7b6ecfbc9253d28b01 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Thu, 3 Apr 2025 16:03:31 +0200 Subject: [PATCH 08/88] Merge with origin --- .../datamanager/ro_crate/preview/DefaultPreviewGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java index 581057d5..909a7b72 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java @@ -163,7 +163,7 @@ public static void generatePreview(String metadata, java.io.File folder) throws TemplateOutput output = new StringOutput(); templateEngine.render("default_preview.jte", input, output); System.out.println(output.toString()); - FileWriter w = new FileWriter(new java.io.File("ro-crate-preview.html")); + FileWriter w = new FileWriter(new java.io.File(folder, "ro-crate-preview.html")); w.write(output.toString()); w.flush(); w.close(); From f3aef3ff008d330b2796cea094f69302969a6fd8 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Sun, 6 Apr 2025 21:23:27 +0200 Subject: [PATCH 09/88] Added json-ld expander to prepare crate metadata for template rendering, started working on proper template for FreeMarker library --- build.gradle | 4 +- .../preview/DefaultPreviewGenerator.java | 126 +++++++++++++++--- .../ro_crate/preview/PreviewGenerator.java | 4 +- .../ro_crate/util/JsonLdExpander.java | 115 ++++++++++++++++ .../ro_crate/writer/FolderWriter.java | 2 +- .../resources/templates/default_preview.frm | 105 +++++++++++++++ .../resources/templates/extended_preview.frm | 87 ++++++++++++ .../crate/preview/PreviewCrateTest.java | 2 +- 8 files changed, 418 insertions(+), 27 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java create mode 100644 src/main/resources/templates/default_preview.frm create mode 100644 src/main/resources/templates/extended_preview.frm diff --git a/build.gradle b/build.gradle index 26715abf..cab24ef1 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ plugins { id 'java' id 'jvm-test-suite' - + id 'application' id 'jacoco' // Adds coveralls task for CI to send results to the coveralls service. id "com.github.kt3k.coveralls" version "2.12.2" @@ -68,7 +68,7 @@ dependencies { implementation 'org.glassfish:jakarta.json:2.0.1' //JTE for template processing implementation('gg.jte:jte:3.1.16') - + implementation("org.freemarker:freemarker:2.3.34") } logging.captureStandardOutput LogLevel.INFO diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java index 909a7b72..6e1dd2f5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java @@ -4,23 +4,24 @@ */ package edu.kit.datamanager.ro_crate.preview; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import gg.jte.CodeResolver; -import gg.jte.ContentType; -import gg.jte.TemplateEngine; -import gg.jte.TemplateOutput; -import gg.jte.output.StringOutput; -import gg.jte.resolve.ResourceCodeResolver; -import java.io.ByteArrayInputStream; -import java.io.FileWriter; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.util.JsonLdExpander; +import static edu.kit.datamanager.ro_crate.util.JsonLdExpander.expandAndPrune; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import freemarker.template.TemplateExceptionHandler; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.model.ZipParameters; /** * @@ -33,6 +34,19 @@ public static class ROCrateModel { public ROCrate crate; public List datasets; public List files; + + public ROCrate getCrate() { + return crate; + } + + public List getDatasets() { + return datasets; + } + + public List getFiles() { + return files; + } + } public static class ROCrate { @@ -43,6 +57,31 @@ public static class ROCrate { public String license; public String datePublished; public List hasPart; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getType() { + return type; + } + + public String getLicense() { + return license; + } + + public String getDatePublished() { + return datePublished; + } + + public List getHasPart() { + return hasPart; + } + } public static class Part { @@ -67,16 +106,29 @@ public static class File { public String encodingFormat; } - public static ROCrateModel mapFromJson(String metadata) throws IOException { - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode root = objectMapper.readTree(metadata); - JsonNode graph = root.get("@graph"); + public static Map mapFromJson(String metadata) throws IOException { + + ObjectMapper mapper = new ObjectMapper(); + + // Read JSON File + Map root = mapper.readValue(metadata, new TypeReference>() { + }); + + // Extract `@graph` as a list of maps + List> graph = (List>) root.get("@graph"); + + // Rename `@graph` to `graph` before passing to FreeMarker + root.put("graph", graph); + root.remove("@graph"); - ROCrate crate = new ROCrate(); + return root; + + /* ROCrate crate = new ROCrate(); List datasets = new ArrayList<>(); List files = new ArrayList<>(); if (graph.isArray()) { + for (JsonNode node : graph) { String id = node.get("@id").asText(); String type = node.get("@type").asText(); @@ -133,11 +185,12 @@ public static ROCrateModel mapFromJson(String metadata) throws IOException { model.crate = crate; model.datasets = datasets; model.files = files; - return model; + //return model; + return null;*/ } public static void generatePreview(String metadata, ZipFile zip) throws IOException { - ROCrateModel model = mapFromJson(metadata); + /* ROCrateModel model = mapFromJson(metadata); Map input = new HashMap(); input.put("model", model); @@ -148,11 +201,11 @@ public static void generatePreview(String metadata, ZipFile zip) throws IOExcept templateEngine.render("default_preview.jte", input, output); ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-preview.html"); - zip.addStream(new ByteArrayInputStream(output.toString().getBytes()), zipParameters); + zip.addStream(new ByteArrayInputStream(output.toString().getBytes()), zipParameters);*/ } public static void generatePreview(String metadata, java.io.File folder) throws IOException { - ROCrateModel model = mapFromJson(metadata); + /* ROCrateModel model = mapFromJson(metadata); Map input = new HashMap(); input.put("model", model); @@ -166,8 +219,41 @@ public static void generatePreview(String metadata, java.io.File folder) throws FileWriter w = new FileWriter(new java.io.File(folder, "ro-crate-preview.html")); w.write(output.toString()); w.flush(); - w.close(); + w.close();*/ + // Create your Configuration instance, and specify if up to what FreeMarker +// version (here 2.3.34) do you want to apply the fixes that are not 100% +// backward-compatible. See the Configuration JavaDoc for details. + Configuration cfg = new Configuration(); + +// Specify the source where the template files come from. Here I set a +// plain directory for it, but non-file-system sources are possible too: + cfg.setClassForTemplateLoading(DefaultPreviewGenerator.class, "/");//DirectoryForTemplateLoading(new File("/where/you/store/templates")); + +// From here we will set the settings recommended for new projects. These +// aren't the defaults for backward compatibilty. +// Set the preferred charset template files are stored in. UTF-8 is +// a good choice in most applications: + cfg.setDefaultEncoding("UTF-8"); +// Sets how errors will appear. +// During web page *development* TemplateExceptionHandler.HTML_DEBUG_HANDLER is better. + cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); + JsonLdExpander ep = new JsonLdExpander(); + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode expanded = ep.expandAndPrune(mapper.readTree(metadata)); + mapper.writerWithDefaultPrettyPrinter().writeValue(System.out, expanded); + Map model = mapper.readValue(mapper.writeValueAsString(expanded), new TypeReference<>() { + }); + System.out.println("MOD " + model); + Template temp = cfg.getTemplate("templates/extended_preview.frm"); + Writer out = new OutputStreamWriter(new FileOutputStream("prev.html")); + + temp.process(model, out); + } catch (Exception ex) { + ex.printStackTrace(); + + } } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java index b348c565..87fe911a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java @@ -12,7 +12,7 @@ */ public class PreviewGenerator { - private static final String command = "rochtml"; + private static final String command = "rochtml1"; public static boolean isRochtmlAvailable() { ProcessBuilder builder = new ProcessBuilder(); @@ -29,9 +29,7 @@ public static boolean isRochtmlAvailable() { Process process; try { process = builder.start(); - int exitVal = process.waitFor(); - System.out.println("EXIT " + exitVal); return exitVal == 0; } catch (InterruptedException | IOException ex) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java new file mode 100644 index 00000000..81bb7101 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java @@ -0,0 +1,115 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.kit.datamanager.ro_crate.util; + +/** + * + * @author jejkal + */ +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.node.*; + +import java.util.*; + +import java.io.File; + +public class JsonLdExpander { + + public static JsonNode expandAndPrune(File jsonLdFile) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(jsonLdFile); + return expandAndPrune(root); + } + + public static JsonNode expandAndPrune(JsonNode root) throws Exception { + ArrayNode graph = (ArrayNode) root.get("@graph"); + ObjectMapper mapper = new ObjectMapper(); + // Index all items by @id + Map idMap = new HashMap<>(); + for (JsonNode node : graph) { + if (node.has("@id")) { + idMap.put(node.get("@id").asText(), node); + } + } + + // Track which ids were directly referenced and expanded + Set expandedIds = new HashSet<>(); + + ArrayNode expandedGraph = mapper.createArrayNode(); + for (JsonNode node : graph) { + // Include only if it's NOT a referenced/expanded object + if (node.has("@id") && expandedIds.contains(node.get("@id").asText())) { + continue; // skip referenced/expanded nodes + } + + JsonNode expandedNode = expandNode(node, idMap, expandedIds, mapper, new HashSet<>()); + expandedGraph.add(expandedNode); + } + + // Rebuild root + ObjectNode newRoot = mapper.createObjectNode(); + newRoot.set("@context", root.get("@context")); + newRoot.set("@graph", expandedGraph); + return newRoot; + } + + private static JsonNode expandNode(JsonNode node, Map idMap, Set expandedIds, ObjectMapper mapper, Set visited) { + if (!node.isObject()) { + return node; + } + + ObjectNode result = mapper.createObjectNode(); + Iterator> fields = node.fields(); + + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + String key = entry.getKey(); + JsonNode value = entry.getValue(); + + if (value.isObject() && value.has("@id")) { + String refId = value.get("@id").asText(); + if (!visited.contains(refId) && idMap.containsKey(refId)) { + visited.add(refId); + expandedIds.add(refId); + result.set(key, expandNode(idMap.get(refId), idMap, expandedIds, mapper, new HashSet<>(visited))); + } else { + result.set(key, value); + } + } else if (value.isArray()) { + ArrayNode newArray = mapper.createArrayNode(); + for (JsonNode element : value) { + if (element.isObject() && element.has("@id")) { + String refId = element.get("@id").asText(); + if (!visited.contains(refId) && idMap.containsKey(refId)) { + visited.add(refId); + expandedIds.add(refId); + newArray.add(expandNode(idMap.get(refId), idMap, expandedIds, mapper, new HashSet<>(visited))); + } else { + newArray.add(element); + } + } else { + newArray.add(expandNode(element, idMap, expandedIds, mapper, visited)); + } + } + result.set(key, newArray); + } else if (value.isObject()) { + result.set(key, expandNode(value, idMap, expandedIds, mapper, visited)); + } else { + result.set(key, value); + } + } + + return result; + } + + // Example usage + public static void main(String[] args) throws Exception { + File file = new File("E:\\Software\\NetbeansProjects\\ro-crate-java\\src\\test\\resources\\crates\\workflowhub\\workflow1\\ro-crate-metadata.json"); + JsonNode expanded = expandAndPrune(file); + + ObjectMapper mapper = new ObjectMapper(); + mapper.writerWithDefaultPrettyPrinter().writeValue(System.out, expanded); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java index 5be66701..ab704181 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java @@ -38,7 +38,7 @@ public void save(Crate crate, String destination) { File json = new File(destination, "ro-crate-metadata.json"); FileUtils.copyInputStreamToFile(inputStream, json); - inputStream.close(); + inputStream.close(); // save also the preview files to the crate destination if (crate.getPreview() != null) { crate.getPreview().saveAllToFolder(file); diff --git a/src/main/resources/templates/default_preview.frm b/src/main/resources/templates/default_preview.frm new file mode 100644 index 00000000..8b253c1b --- /dev/null +++ b/src/main/resources/templates/default_preview.frm @@ -0,0 +1,105 @@ + + + + + + ${crate.name!"RO-Crate Metadata"} + + + + +
+

${name!"Untitled Dataset"}

+

Metadata

+
+ <#if crate.description??> +
Description: ${crate.description}
+ + <#if crate.license??> + + + <#if crate.datePublished??> +
Published: ${crate.datePublished}
+ +
+ +

Contained Files

+
+
    + <#list crate.hasPart as part> +
  • ${part['@id']} + <#list files as item> + <#if item['@id'] == part['@id']> +
      + <#if item.name??> +
    • Name: ${item.name}
      • + + <#if item.description??> +
    • Description: ${item.description}
      • + + <#if item.contentSize??> +
    • Size: ${item.contentSize}
      • + + <#if item.encodingFormat??> +
    • Format: ${item.encodingFormat}
      • + +
    + + + <#list datasets as item> + <#if item['@id'] == part['@id']> +
      + <#if item.name??> +
    • Name: ${item.name}
      • + + <#if item.description??> +
    • Description: ${item.description}
      • + + <#if item.contentSize??> +
    • Size: ${item.contentSize}
      • + + <#if item.encodingFormat??> +
    • Format: ${item.encodingFormat}
      • + +
    + + +
    • + +
+
+
+ + \ No newline at end of file diff --git a/src/main/resources/templates/extended_preview.frm b/src/main/resources/templates/extended_preview.frm new file mode 100644 index 00000000..22e2efbb --- /dev/null +++ b/src/main/resources/templates/extended_preview.frm @@ -0,0 +1,87 @@ + + + + + ${about.name!"Research Object Crate"} + + + +<#-- Find main entry with @type = CreativeWork and 'about' property --> +<#assign main = graph?filter(i -> i["@type"]?contains("CreativeWork") && i["about"]??)?first> +<#assign about = main.about> + +
+

${about.name! "Untitled Dataset"}

+
+
Date Published: ${about.datePublished! "N/A"}
+
License: ${about.license! "N/A"}
+
Based On: ${about.isBasedOn! "N/A"}
+
+
+ +
+

Contained Items

+ + <#-- Render hasPart section --> + <#if about.hasPart??> + <#list about.hasPart as part> +
+

${part.name! part["@id"]}

+
Type: ${part["@type"]!}
+ <#if part.version??> +
Version: ${part.version}
+ + <#if part.url??> +
URL: ${part.url}
+ + <#if part.creator??> +
Creator: + <#list part.creator as c> + ${c.name! "Unknown"}<#if c_has_next>, + +
+ + <#if part.programmingLanguage??> +
Language: ${part.programmingLanguage.name!}
+ +
+ + + + <#-- Render mentions --> + <#if about.mentions??> +

Mentions

+ <#list about.mentions as m> +
+

${m.name! m["@id"]}

+
Type: ${m["@type"]!}
+ <#if m.definition??> +
Definition: ${m.definition["@id"]!}
+ + <#if m.mainEntity??> +
Main Entity: ${m.mainEntity["@id"]!}
+ + <#if m.instance??> + <#list m.instance as i> +
Instance: ${i.name!} (${i.url!})
+ <#if i.runsOn??> +
Runs On: ${i.runsOn.name!} - ${i.runsOn.url["@id"]!}
+ + + +
+ + +
+ + diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index bd7b9156..a01a915d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -28,7 +28,7 @@ void testAutomaticPreview(@TempDir Path temp) { .build(); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); writer.save(crate, location.toFile().getAbsolutePath()); - + writer.save(crate, "."); assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); } From 9e41a50e99f1bcae018de492198e43e368c79ed3 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Mon, 7 Apr 2025 15:59:47 +0200 Subject: [PATCH 10/88] Finished template and preview generation, still needs to be integrated into crate builder process properly --- .../edu/kit/datamanager/ro_crate/RoCrate.java | 24 +- .../ro_crate/preview/CustomPreview.java | 143 +++++++++--- .../preview/DefaultPreviewGenerator.java | 220 +++++------------- .../ro_crate/preview/PreviewGenerator.java | 2 +- .../preview/model/ROCratePreviewModel.java | 126 ++++++++++ .../ro_crate/util/JsonLdExpander.java | 9 - .../resources/templates/custom_preview.ftl | 194 +++++++++++++++ .../resources/templates/default_preview.frm | 105 --------- .../resources/templates/default_preview.jte | 54 ----- .../resources/templates/extended_preview.frm | 87 ------- 10 files changed, 495 insertions(+), 469 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/preview/model/ROCratePreviewModel.java create mode 100644 src/main/resources/templates/custom_preview.ftl delete mode 100644 src/main/resources/templates/default_preview.frm delete mode 100644 src/main/resources/templates/default_preview.jte delete mode 100644 src/main/resources/templates/extended_preview.frm diff --git a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java index ea6911e0..4b1bc103 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java @@ -184,8 +184,8 @@ public AbstractEntity getEntityById(String id) { /** * {@inheritDoc} *

- * Note: This will also link the DataEntity to the root node - * using the root nodes hasPart property. + * Note: This will also link the DataEntity to the root node using the root + * nodes hasPart property. * * @param entity the DataEntity to add to this crate. */ @@ -335,8 +335,8 @@ public RoCrateBuilder addDescription(String description) { /** * Adds a data entity to the crate. *

- * Note: This will also link the DataEntity to the root node - * using the root nodes hasPart property. + * Note: This will also link the DataEntity to the root node using the + * root nodes hasPart property. * * @param dataEntity the DataEntity to add to this crate. * @return returns the builder for further usage. @@ -372,21 +372,22 @@ public RoCrateBuilder setLicense(ContextualEntity license) { /** * Setting the license of the crate using only a license identifier. * - * @param licenseId the licenses identifier. Should be a resolveable URI. + * @param licenseId the licenses identifier. Should be a resolveable + * URI. * @return the builder */ public RoCrateBuilder setLicense(String licenseId) { ContextualEntity licenseEntity = new ContextualEntity.ContextualEntityBuilder() - .setId(licenseId) - .build(); + .setId(licenseId) + .build(); this.setLicense(licenseEntity); return this; } /** - * Adds a property with date time format. The property should match the ISO 8601 - * date format. - * + * Adds a property with date time format. The property should match the + * ISO 8601 date format. + * * @param dateValue time string in ISO 8601 format * @return this builder * @throws IllegalArgumentException if format is not ISO 8601 @@ -458,7 +459,8 @@ public BuilderWithDraftFeatures(String name, String description, String datePubl } /** - * @see RoCrateBuilder#RoCrateBuilder(String, String, String, ContextualEntity) + * @see RoCrateBuilder#RoCrateBuilder(String, String, String, + * ContextualEntity) */ public BuilderWithDraftFeatures(String name, String description, String datePublished, ContextualEntity licenseId) { super(name, description, datePublished, licenseId); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index 0139c031..c5ef2770 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -1,7 +1,23 @@ package edu.kit.datamanager.ro_crate.preview; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import static edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.mapFromJson; +import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateExceptionHandler; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.model.ZipParameters; @@ -16,53 +32,112 @@ */ public class CustomPreview implements CratePreview { - private final File metadataHtml; - private final File otherFiles; + private final Configuration cfg; - public CustomPreview(File metadataHtml, File otherFiles) { - this.metadataHtml = metadataHtml; - this.otherFiles = otherFiles; - } + public CustomPreview() { + cfg = new Configuration(Configuration.VERSION_2_3_34); + cfg.setClassForTemplateLoading(DefaultPreviewGenerator.class, "/"); + cfg.setDefaultEncoding("UTF-8"); + cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); - public CustomPreview(File metadataHtml) { - this.metadataHtml = metadataHtml; - this.otherFiles = null; } - @Override - public void saveAllToZip(ZipFile zipFile) { - if (this.metadataHtml != null) { - try { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-preview.html"); - zipFile.addFile(this.metadataHtml, zipParameters); - } catch (ZipException e) { - System.err.println("Exception writing preview html to zip"); + public ROCratePreviewModel mapFromJson(String metadata) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = (JsonNode) mapper.readValue(metadata, JsonNode.class); + JsonNode graph = root.get("@graph"); + ROCratePreviewModel.ROCrate crate = new ROCratePreviewModel.ROCrate(); + List datasets = new ArrayList<>(); + List files = new ArrayList<>(); + + if (graph.isArray()) { + + for (JsonNode node : graph) { + String id = node.get("@id").asText(); + List types = new LinkedList<>(); + if (node.get("@type").isArray()) { + + Collections.addAll(types, (String[]) mapper.convertValue(node.get("@type"), String[].class)); + } else { + types.add(node.get("@type").asText()); + } + + if (types.contains("Dataset") && "./".equals(id)) { + crate.name = node.get("name").asText(); + crate.description = node.get("description") == null ? null : node.get("description").asText(); + crate.type = "Dataset"; + crate.license = node.get("license").isObject() ? node.get("license").get("@id").asText() : node.get("license").asText(); + crate.datePublished = node.get("datePublished") == null ? null : node.get("datePublished").asText(); + crate.hasPart = new ArrayList<>(); + + if (node.has("hasPart")) { + for (JsonNode part : node.get("hasPart")) { + ROCratePreviewModel.Part p = new ROCratePreviewModel.Part(); + p.id = part.get("@id").asText(); + p.name = part.get("@id").asText(); // Name will be replaced later + crate.hasPart.add(p); + } + + } + } else if (types.contains("Dataset")) { + ROCratePreviewModel.Dataset dataset = new ROCratePreviewModel.Dataset(); + dataset.id = id; + dataset.name = node.get("name").asText(); + dataset.description = node.get("description").asText(); + datasets.add(dataset); + } else if (types.contains("File")) { + ROCratePreviewModel.File file = new ROCratePreviewModel.File(); + file.id = id; + file.name = node.get("name") == null ? null : node.get("name").asText(); + file.description = node.get("description") == null ? null : node.get("description").asText(); + file.contentSize = node.get("contentSize") == null ? null : node.get("contentSize").asText(); + file.encodingFormat = node.get("encodingFormat") == null ? null : node.get("encodingFormat").asText(); + files.add(file); + } } } - if (this.otherFiles != null) { - try { - zipFile.addFolder(this.otherFiles); - zipFile.renameFile(this.otherFiles.getName() + "/", "ro-crate-preview_files/"); - } catch (ZipException e) { - System.err.println("Exception writing preview files to zip"); + + // Update Part names using dataset and file lists + if (crate.hasPart != null) { + for (ROCratePreviewModel.Part part : crate.hasPart) { + for (ROCratePreviewModel.Dataset dataset : datasets) { + if (dataset.id.equals(part.id) && dataset.name != null) { + part.name = dataset.name; + } + } + for (ROCratePreviewModel.File file : files) { + if (file.id.equals(part.id) && file.name != null) { + part.name = file.name; + } + } } } + + ROCratePreviewModel model = new ROCratePreviewModel(); + model.crate = crate; + model.datasets = datasets; + model.files = files; + return model; + } + + @Override + public void saveAllToZip(ZipFile zipFile) { + } @Override public void saveAllToFolder(File folder) { + try { - if (this.metadataHtml != null) { - File fileInCrate = folder.toPath().resolve("ro-crate-preview.html").toFile(); - FileUtils.copyFile(this.metadataHtml, fileInCrate); - } - if (this.otherFiles != null) { - File folderName = folder.toPath().resolve("ro-crate-preview_files").toFile(); - FileUtils.copyDirectory(this.otherFiles, folderName); - } - } catch (IOException e) { - e.printStackTrace(); + Map dataModel = new HashMap<>(); + dataModel.put("crateModel", mapFromJson(metadata)); + Template temp = cfg.getTemplate("templates/custom_preview.frm"); + Writer out = new OutputStreamWriter(new FileOutputStream("prev.html")); + + temp.process(dataModel, out); + } catch (Exception ex) { + ex.printStackTrace(); + } } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java index 6e1dd2f5..e0ee48b6 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java @@ -1,24 +1,23 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ package edu.kit.datamanager.ro_crate.preview; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel; +import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel.Dataset; +import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel.Part; +import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel.ROCrate; import edu.kit.datamanager.ro_crate.util.JsonLdExpander; -import static edu.kit.datamanager.ro_crate.util.JsonLdExpander.expandAndPrune; import freemarker.template.Configuration; import freemarker.template.Template; -import freemarker.template.TemplateException; import freemarker.template.TemplateExceptionHandler; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import net.lingala.zip4j.ZipFile; @@ -29,164 +28,82 @@ */ public class DefaultPreviewGenerator { - public static class ROCrateModel { - - public ROCrate crate; - public List datasets; - public List files; - - public ROCrate getCrate() { - return crate; - } - - public List getDatasets() { - return datasets; - } - - public List getFiles() { - return files; - } - - } - - public static class ROCrate { - - public String name; - public String description; - public String type; - public String license; - public String datePublished; - public List hasPart; - - public String getName() { - return name; - } - - public String getDescription() { - return description; - } - - public String getType() { - return type; - } - - public String getLicense() { - return license; - } - - public String getDatePublished() { - return datePublished; - } - - public List getHasPart() { - return hasPart; - } - - } - - public static class Part { - - public String id; - public String name; - } - - public static class Dataset { - - public String id; - public String name; - public String description; - } - - public static class File { - - public String id; - public String name; - public String description; - public String contentSize; - public String encodingFormat; - } - - public static Map mapFromJson(String metadata) throws IOException { - + public static ROCratePreviewModel mapFromJson(String metadata) throws IOException { ObjectMapper mapper = new ObjectMapper(); - - // Read JSON File - Map root = mapper.readValue(metadata, new TypeReference>() { - }); - - // Extract `@graph` as a list of maps - List> graph = (List>) root.get("@graph"); - - // Rename `@graph` to `graph` before passing to FreeMarker - root.put("graph", graph); - root.remove("@graph"); - - return root; - - /* ROCrate crate = new ROCrate(); + JsonNode root = (JsonNode) mapper.readValue(metadata, JsonNode.class); + JsonNode graph = root.get("@graph"); + ROCrate crate = new ROCrate(); List datasets = new ArrayList<>(); - List files = new ArrayList<>(); + List files = new ArrayList<>(); if (graph.isArray()) { for (JsonNode node : graph) { String id = node.get("@id").asText(); - String type = node.get("@type").asText(); + List types = new LinkedList<>(); + if (node.get("@type").isArray()) { + + Collections.addAll(types, (String[]) mapper.convertValue(node.get("@type"), String[].class)); + } else { + types.add(node.get("@type").asText()); + } - if ("Dataset".equals(type) && "./".equals(id)) { + if (types.contains("Dataset") && "./".equals(id)) { crate.name = node.get("name").asText(); - crate.description = node.get("description").asText(); - crate.type = type; - crate.license = node.get("license").get("@id").asText(); - crate.datePublished = node.get("datePublished").asText(); + crate.description = node.get("description") == null ? null : node.get("description").asText(); + crate.type = "Dataset"; + crate.license = node.get("license").isObject() ? node.get("license").get("@id").asText() : node.get("license").asText(); + crate.datePublished = node.get("datePublished") == null ? null : node.get("datePublished").asText(); crate.hasPart = new ArrayList<>(); if (node.has("hasPart")) { for (JsonNode part : node.get("hasPart")) { Part p = new Part(); p.id = part.get("@id").asText(); - p.name = "Unknown"; // Name will be set later + p.name = part.get("@id").asText(); // Name will be replaced later crate.hasPart.add(p); } + } - } else if ("Dataset".equals(type)) { + } else if (types.contains("Dataset")) { Dataset dataset = new Dataset(); dataset.id = id; dataset.name = node.get("name").asText(); dataset.description = node.get("description").asText(); datasets.add(dataset); - } else if ("File".equals(type)) { - File file = new File(); + } else if (types.contains("File")) { + ROCratePreviewModel.File file = new ROCratePreviewModel.File(); file.id = id; - file.name = node.get("name").asText(); - file.description = node.get("description").asText(); - file.contentSize = node.get("contentSize").asText(); - file.encodingFormat = node.get("encodingFormat").asText(); + file.name = node.get("name") == null ? null : node.get("name").asText(); + file.description = node.get("description") == null ? null : node.get("description").asText(); + file.contentSize = node.get("contentSize") == null ? null : node.get("contentSize").asText(); + file.encodingFormat = node.get("encodingFormat") == null ? null : node.get("encodingFormat").asText(); files.add(file); } } } // Update Part names using dataset and file lists - for (Part part : crate.hasPart) { - for (Dataset dataset : datasets) { - if (dataset.id.equals(part.id)) { - part.name = dataset.name; + if (crate.hasPart != null) { + for (Part part : crate.hasPart) { + for (Dataset dataset : datasets) { + if (dataset.id.equals(part.id) && dataset.name != null) { + part.name = dataset.name; + } } - } - for (File file : files) { - if (file.id.equals(part.id)) { - part.name = file.name; + for (ROCratePreviewModel.File file : files) { + if (file.id.equals(part.id) && file.name != null) { + part.name = file.name; + } } } } - ROCrateModel model = new ROCrateModel(); + ROCratePreviewModel model = new ROCratePreviewModel(); model.crate = crate; model.datasets = datasets; model.files = files; - //return model; - return null;*/ + return model; } public static void generatePreview(String metadata, ZipFile zip) throws IOException { @@ -205,51 +122,18 @@ public static void generatePreview(String metadata, ZipFile zip) throws IOExcept } public static void generatePreview(String metadata, java.io.File folder) throws IOException { - /* ROCrateModel model = mapFromJson(metadata); - - Map input = new HashMap(); - input.put("model", model); - - CodeResolver codeResolver = new ResourceCodeResolver("templates"); // This is the directory where your .jte files are located. - TemplateEngine templateEngine = TemplateEngine.create(codeResolver, ContentType.Html); - - TemplateOutput output = new StringOutput(); - templateEngine.render("default_preview.jte", input, output); - System.out.println(output.toString()); - FileWriter w = new FileWriter(new java.io.File(folder, "ro-crate-preview.html")); - w.write(output.toString()); - w.flush(); - w.close();*/ - // Create your Configuration instance, and specify if up to what FreeMarker -// version (here 2.3.34) do you want to apply the fixes that are not 100% -// backward-compatible. See the Configuration JavaDoc for details. - Configuration cfg = new Configuration(); - -// Specify the source where the template files come from. Here I set a -// plain directory for it, but non-file-system sources are possible too: - cfg.setClassForTemplateLoading(DefaultPreviewGenerator.class, "/");//DirectoryForTemplateLoading(new File("/where/you/store/templates")); - -// From here we will set the settings recommended for new projects. These -// aren't the defaults for backward compatibilty. -// Set the preferred charset template files are stored in. UTF-8 is -// a good choice in most applications: + Configuration cfg = new Configuration(Configuration.VERSION_2_3_34); + cfg.setClassForTemplateLoading(DefaultPreviewGenerator.class, "/"); cfg.setDefaultEncoding("UTF-8"); - -// Sets how errors will appear. -// During web page *development* TemplateExceptionHandler.HTML_DEBUG_HANDLER is better. cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); - JsonLdExpander ep = new JsonLdExpander(); - ObjectMapper mapper = new ObjectMapper(); + try { - JsonNode expanded = ep.expandAndPrune(mapper.readTree(metadata)); - mapper.writerWithDefaultPrettyPrinter().writeValue(System.out, expanded); - Map model = mapper.readValue(mapper.writeValueAsString(expanded), new TypeReference<>() { - }); - System.out.println("MOD " + model); - Template temp = cfg.getTemplate("templates/extended_preview.frm"); + Map dataModel = new HashMap<>(); + dataModel.put("crateModel", mapFromJson(metadata)); + Template temp = cfg.getTemplate("templates/custom_preview.frm"); Writer out = new OutputStreamWriter(new FileOutputStream("prev.html")); - temp.process(model, out); + temp.process(dataModel, out); } catch (Exception ex) { ex.printStackTrace(); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java index 87fe911a..d170341c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java @@ -12,7 +12,7 @@ */ public class PreviewGenerator { - private static final String command = "rochtml1"; + private static final String command = "rochtml"; public static boolean isRochtmlAvailable() { ProcessBuilder builder = new ProcessBuilder(); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/model/ROCratePreviewModel.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/model/ROCratePreviewModel.java new file mode 100644 index 00000000..6750784f --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/model/ROCratePreviewModel.java @@ -0,0 +1,126 @@ +package edu.kit.datamanager.ro_crate.preview.model; + +import java.util.List; + +/** + * + * @author jejkal + */ +public class ROCratePreviewModel { + + public ROCrate crate; + public List datasets; + public List files; + + public ROCrate getCrate() { + return crate; + } + + public List getDatasets() { + return datasets; + } + + public List getFiles() { + return files; + } + + public static class ROCrate { + + public String name; + public String description; + public String type; + public String license; + public String datePublished; + public List hasPart; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getType() { + return type; + } + + public String getLicense() { + return license; + } + + public String getDatePublished() { + return datePublished; + } + + public List getHasPart() { + return hasPart; + } + + } + + public static class Part { + + public String id; + public String name; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + } + + public static class Dataset { + + public String id; + public String name; + public String description; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + } + + public static class File { + + public String id; + public String name; + public String description; + public String contentSize; + public String encodingFormat; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getContentSize() { + return contentSize; + } + + public String getEncodingFormat() { + return encodingFormat; + } + + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java index 81bb7101..d6b7e987 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java @@ -103,13 +103,4 @@ private static JsonNode expandNode(JsonNode node, Map idMap, S return result; } - - // Example usage - public static void main(String[] args) throws Exception { - File file = new File("E:\\Software\\NetbeansProjects\\ro-crate-java\\src\\test\\resources\\crates\\workflowhub\\workflow1\\ro-crate-metadata.json"); - JsonNode expanded = expandAndPrune(file); - - ObjectMapper mapper = new ObjectMapper(); - mapper.writerWithDefaultPrettyPrinter().writeValue(System.out, expanded); - } } diff --git a/src/main/resources/templates/custom_preview.ftl b/src/main/resources/templates/custom_preview.ftl new file mode 100644 index 00000000..d19b57bc --- /dev/null +++ b/src/main/resources/templates/custom_preview.ftl @@ -0,0 +1,194 @@ + + + + + ${crateModel.crate.name!} + + + + + +

+

${crateModel.crate.name!}

+ + <#if crateModel.crate.description??> +

Description: ${crateModel.crate.description}

+ + + <#if crateModel.crate.license??> +

License: + <#if crateModel.crate.license?starts_with("http")> + ${crateModel.crate.license} + <#else> + ${crateModel.crate.license} + +

+ + + <#if crateModel.crate.datePublished??> +

Date Published: ${crateModel.crate.datePublished}

+ +
+ +
+ + <#if crateModel.crate.hasPart?? && crateModel.crate.hasPart?size gt 0> +
+

Parts

+
    + <#list crateModel.crate.hasPart as part> +
  • + ${part.name!} + <#if part.id??> + - ${part.id} + +
    • + +
+
+ + + <#if crateModel.datasets?? && crateModel.datasets?size gt 0> +
+

Datasets

+
    + <#list crateModel.datasets as dataset> +
  • + ${dataset.name!} + <#if dataset.id??> + - ${dataset.id} + + <#if dataset.description??> +
    ${dataset.description} + +
    • + +
+
+ + + <#if crateModel.files?? && crateModel.files?size gt 0> +
+

Files

+
    + <#list crateModel.files as file> +
  • + ${file.name!} + <#if file.id??> + - ${file.id} + + <#if file.description??> +
    ${file.description} + + <#if file.contentSize??> +
    Size: ${file.contentSize} + + <#if file.encodingFormat??> +
    Format: ${file.encodingFormat} + +
    • + +
+
+ + +
+ + + \ No newline at end of file diff --git a/src/main/resources/templates/default_preview.frm b/src/main/resources/templates/default_preview.frm deleted file mode 100644 index 8b253c1b..00000000 --- a/src/main/resources/templates/default_preview.frm +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - ${crate.name!"RO-Crate Metadata"} - - - - -
-

${name!"Untitled Dataset"}

-

Metadata

-
- <#if crate.description??> -
Description: ${crate.description}
- - <#if crate.license??> - - - <#if crate.datePublished??> -
Published: ${crate.datePublished}
- -
- -

Contained Files

-
-
    - <#list crate.hasPart as part> -
  • ${part['@id']} - <#list files as item> - <#if item['@id'] == part['@id']> -
      - <#if item.name??> -
    • Name: ${item.name}
      • - - <#if item.description??> -
    • Description: ${item.description}
      • - - <#if item.contentSize??> -
    • Size: ${item.contentSize}
      • - - <#if item.encodingFormat??> -
    • Format: ${item.encodingFormat}
      • - -
    - - - <#list datasets as item> - <#if item['@id'] == part['@id']> -
      - <#if item.name??> -
    • Name: ${item.name}
      • - - <#if item.description??> -
    • Description: ${item.description}
      • - - <#if item.contentSize??> -
    • Size: ${item.contentSize}
      • - - <#if item.encodingFormat??> -
    • Format: ${item.encodingFormat}
      • - -
    - - -
    • - -
-
-
- - \ No newline at end of file diff --git a/src/main/resources/templates/default_preview.jte b/src/main/resources/templates/default_preview.jte deleted file mode 100644 index 85ffd2b8..00000000 --- a/src/main/resources/templates/default_preview.jte +++ /dev/null @@ -1,54 +0,0 @@ -@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.ROCrateModel -@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.Part -@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.Dataset -@import edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.File - -@param ROCrateModel model - - - - - - - RO-Crate Viewer - - - -
-

RO-Crate Metadata

-
-

${model.crate.name}

-

Description: ${model.crate.description}

-

Type: ${model.crate.type}

-

License: License

-

Date Published: ${model.crate.datePublished}

-

Contains:

-
    - @for(Part part : model.crate.hasPart) { -
  • ${part.name}
    • - @endfor} -
-
- - @for(Dataset dataset : model.datasets) { -
-

Dataset: ${dataset.name}

-

Description: ${dataset.description}

-
- @endfor} - - @if(!model.files.isEmpty()){ - @for(File file : model.files) { -
-

File: ${file.name}

-

Description: ${file.description}

-

Size: ${file.contentSize} bytes

-

Format: ${file.encodingFormat}

-
- @endfor} - }@else

No files.

- @endif -
- - - diff --git a/src/main/resources/templates/extended_preview.frm b/src/main/resources/templates/extended_preview.frm deleted file mode 100644 index 22e2efbb..00000000 --- a/src/main/resources/templates/extended_preview.frm +++ /dev/null @@ -1,87 +0,0 @@ - - - - - ${about.name!"Research Object Crate"} - - - -<#-- Find main entry with @type = CreativeWork and 'about' property --> -<#assign main = graph?filter(i -> i["@type"]?contains("CreativeWork") && i["about"]??)?first> -<#assign about = main.about> - -
-

${about.name! "Untitled Dataset"}

-
-
Date Published: ${about.datePublished! "N/A"}
-
License: ${about.license! "N/A"}
-
Based On: ${about.isBasedOn! "N/A"}
-
-
- -
-

Contained Items

- - <#-- Render hasPart section --> - <#if about.hasPart??> - <#list about.hasPart as part> -
-

${part.name! part["@id"]}

-
Type: ${part["@type"]!}
- <#if part.version??> -
Version: ${part.version}
- - <#if part.url??> -
URL: ${part.url}
- - <#if part.creator??> -
Creator: - <#list part.creator as c> - ${c.name! "Unknown"}<#if c_has_next>, - -
- - <#if part.programmingLanguage??> -
Language: ${part.programmingLanguage.name!}
- -
- - - - <#-- Render mentions --> - <#if about.mentions??> -

Mentions

- <#list about.mentions as m> -
-

${m.name! m["@id"]}

-
Type: ${m["@type"]!}
- <#if m.definition??> -
Definition: ${m.definition["@id"]!}
- - <#if m.mainEntity??> -
Main Entity: ${m.mainEntity["@id"]!}
- - <#if m.instance??> - <#list m.instance as i> -
Instance: ${i.name!} (${i.url!})
- <#if i.runsOn??> -
Runs On: ${i.runsOn.name!} - ${i.runsOn.url["@id"]!}
- - - -
- - -
- - From 0e808759a45897b68351f8a6651dcf7b0446eccd Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Tue, 8 Apr 2025 15:22:58 +0200 Subject: [PATCH 11/88] Integrated custom preview, renamed previously CustomPreview to StaticPreview, integrated stream writing to all previews, some tests still failing --- .../ro_crate/entities/data/DataEntity.java | 14 +- .../ro_crate/entities/data/DataSetEntity.java | 44 +----- .../ro_crate/preview/AutomaticPreview.java | 40 +++-- .../ro_crate/preview/CratePreview.java | 17 ++- .../ro_crate/preview/CustomPreview.java | 74 +++++++-- .../preview/DefaultPreviewGenerator.java | 143 ------------------ .../ro_crate/preview/StaticPreview.java | 70 +++++++++ .../datamanager/ro_crate/util/ZipUtil.java | 55 +++++++ .../ro_crate/writer/ZipStreamWriter.java | 9 +- .../ro_crate/writer/ZipWriter.java | 1 - .../ro_crate/crate/ReadAndWriteTest.java | 3 +- .../crate/preview/PreviewCrateTest.java | 25 ++- .../ro_crate/preview/PreviewTest.java | 53 ++++++- 13 files changed, 297 insertions(+), 251 deletions(-) delete mode 100644 src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 1b2377a3..50ca8d2a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -5,6 +5,7 @@ import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl; +import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; import java.io.FileInputStream; @@ -68,7 +69,6 @@ public void saveToZip(ZipFile zipFile) throws ZipException { if (this.path != null) { ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip(this.getId()); - System.out.println("ADD FILE " + this.path.toFile()); zipFile.addFile(this.path.toFile(), zipParameters); } } @@ -84,17 +84,7 @@ public void saveToZip(ZipFile zipFile) throws ZipException { */ public void saveToStream(ZipOutputStream zipStream) throws ZipException, IOException { if (this.path != null) { - byte[] buff = new byte[4096]; - int readLen; - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(this.getId()); - zipStream.putNextEntry(zipParameters); - try (InputStream inputStream = new FileInputStream(this.path.toFile())) { - while ((readLen = inputStream.read(buff)) != -1) { - zipStream.write(buff, 0, readLen); - } - } - zipStream.closeEntry(); + ZipUtil.addFileToZipStream(zipStream, this.path.toFile(), this.getId()); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index ff3ef0bc..97d302f7 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -4,17 +4,14 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import edu.kit.datamanager.ro_crate.entities.serializers.HasPartSerializer; -import java.io.File; -import java.io.FileInputStream; +import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.IOException; -import java.io.InputStream; import java.util.HashSet; import java.util.Set; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; -import net.lingala.zip4j.model.ZipParameters; /** * A helping class for the creating of Data entities of type Dataset. @@ -55,47 +52,10 @@ public void saveToZip(ZipFile zipFile) throws ZipException { @Override public void saveToStream(ZipOutputStream zipOutputStream) throws ZipException, IOException { if (this.getPath() != null) { - addFolderToZip(zipOutputStream, this.getPath().toAbsolutePath().toString(), this.getPath().getFileName().toString()); + ZipUtil.addFolderToZipStream(zipOutputStream, this.getPath().toAbsolutePath().toString(), this.getPath().getFileName().toString()); } } - public void addFolderToZip(ZipOutputStream zipOutputStream, String folderPath, String parentPath) throws IOException { - File folder = new File(folderPath); - if (!folder.exists() || !folder.isDirectory()) { - throw new IllegalArgumentException("The provided folder path is not a valid directory: " + folderPath); - } - - File[] files = folder.listFiles(); - if (files == null) { - return; - } - - for (File file : files) { - String zipEntryPath = parentPath.isEmpty() ? file.getName() : parentPath + "/" + file.getName(); - if (file.isDirectory()) { - addFolderToZip(zipOutputStream, file.getAbsolutePath(), zipEntryPath); - } else { - addFileToZip(zipOutputStream, file, zipEntryPath); - } - } - } - - private void addFileToZip(ZipOutputStream zipOutputStream, File file, String zipEntryPath) throws IOException { - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip(zipEntryPath); - zipOutputStream.putNextEntry(zipParameters); - - try (InputStream inputStream = new FileInputStream(file)) { - byte[] buffer = new byte[4096]; - int len; - while ((len = inputStream.read(buffer)) != -1) { - zipOutputStream.write(buffer, 0, len); - } - } - - zipOutputStream.closeEntry(); - } - public void addToHasPart(String id) { this.hasPart.add(id); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index 96617e13..c840152f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -1,8 +1,11 @@ package edu.kit.datamanager.ro_crate.preview; +import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; /** @@ -19,34 +22,41 @@ public AutomaticPreview() { } @Override - public void saveAllToZip(ZipFile zipFile) { + public void saveAllToZip(ZipFile zipFile) throws IOException { + // extract the .json file so we can run the "rochtml" tool on it" try { - // extract the .json file so we can run the "rochtml" tool on it" zipFile.extractFile("ro-crate-metadata.json", "temp"); if (PreviewGenerator.isRochtmlAvailable()) { PreviewGenerator.generatePreview("temp"); - } else { - DefaultPreviewGenerator.generatePreview(FileUtils.readFileToString(new File("temp/ro-crate-metadata.json"), "UTF-8"), zipFile); + zipFile.addFile("temp/ro-crate-preview.html"); } - zipFile.addFile("temp/ro-crate-preview.html"); + } finally { FileUtils.deleteDirectory(new File("temp")); - } catch (IOException e) { - e.printStackTrace(); } } @Override - public void saveAllToFolder(File folder) { - + public void saveAllToFolder(File folder) throws IOException { if (PreviewGenerator.isRochtmlAvailable()) { PreviewGenerator.generatePreview(folder.getAbsolutePath()); - } else { - try { - DefaultPreviewGenerator.generatePreview(FileUtils.readFileToString(new File(folder, "ro-crate-metadata.json"), "UTF-8"), folder); - } catch (IOException ex) { - ex.printStackTrace(); - } } + } + @Override + public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { + try { + FileUtils.forceMkdir(new File("temp")); + FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json")); + writer.write(metadata); + writer.flush(); + writer.close(); + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview("temp"); + ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + } + } finally { + FileUtils.deleteDirectory(new File("temp")); + } } + } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java index b828f5f8..14ea76ea 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CratePreview.java @@ -1,19 +1,24 @@ package edu.kit.datamanager.ro_crate.preview; import java.io.File; +import java.io.IOException; import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; /** - * Interface for the ROCrate preview. - * This manages the human-readable representation of a crate. + * Interface for the ROCrate preview. This manages the human-readable + * representation of a crate. * * @author Nikola Tzotchev on 6.2.2022 г. - * @version 1 + * @author jejkal + * @version 2 */ public interface CratePreview { - void saveAllToZip(ZipFile zipFile); + void saveAllToZip(ZipFile zipFile) throws IOException; + + void saveAllToFolder(File folder) throws IOException; + + void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException; - void saveAllToFolder(File folder); - } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index c5ef2770..433de692 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -2,13 +2,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import static edu.kit.datamanager.ro_crate.preview.DefaultPreviewGenerator.mapFromJson; import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel; +import edu.kit.datamanager.ro_crate.util.ZipUtil; import freemarker.template.Configuration; import freemarker.template.Template; +import freemarker.template.TemplateException; import freemarker.template.TemplateExceptionHandler; import java.io.File; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; @@ -19,16 +21,15 @@ import java.util.List; import java.util.Map; import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; /** - * This class represents the custom preview of a crate, which means html files - * created from outside sources. + * This class generates a custom preview without requiring external + * dependencies, i.e., rochtml. Therefore, the FreeMarker template located under + * resources/templates/custom_preview.ftl is used. * - * @author Nikola Tzotchev on 12.2.2022 г. - * @version 1 + * @author jejkal */ public class CustomPreview implements CratePreview { @@ -36,7 +37,7 @@ public class CustomPreview implements CratePreview { public CustomPreview() { cfg = new Configuration(Configuration.VERSION_2_3_34); - cfg.setClassForTemplateLoading(DefaultPreviewGenerator.class, "/"); + cfg.setClassForTemplateLoading(CustomPreview.class, "/"); cfg.setDefaultEncoding("UTF-8"); cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); @@ -121,23 +122,66 @@ public ROCratePreviewModel mapFromJson(String metadata) throws IOException { } @Override - public void saveAllToZip(ZipFile zipFile) { - + public void saveAllToZip(ZipFile zipFile) throws IOException { + zipFile.extractFile("ro-crate-metadata.json", "temp"); + String metadata = FileUtils.readFileToString(new File("temp/ro-crate-metadata.json"), "UTF-8"); + try { + Map dataModel = new HashMap<>(); + dataModel.put("crateModel", mapFromJson(metadata)); + Template temp = cfg.getTemplate("templates/custom_preview.ftl"); + Writer out = new OutputStreamWriter(new FileOutputStream("temp/ro-crate-preview.html")); + temp.process(dataModel, out); + zipFile.addFile("temp/ro-crate-preview.html"); + } catch (TemplateException ex) { + throw new IOException("Failed to generate preview.", ex); + } finally { + try { + FileUtils.deleteDirectory(new File("temp")); + } catch (IOException ex) { + //ignore + } + } } @Override - public void saveAllToFolder(File folder) { - + public void saveAllToFolder(File folder) throws IOException { + String metadata = FileUtils.readFileToString(new File(folder, "ro-crate-metadata.json"), "UTF-8"); try { Map dataModel = new HashMap<>(); dataModel.put("crateModel", mapFromJson(metadata)); - Template temp = cfg.getTemplate("templates/custom_preview.frm"); + Template temp = cfg.getTemplate("templates/custom_preview.ftl"); Writer out = new OutputStreamWriter(new FileOutputStream("prev.html")); temp.process(dataModel, out); - } catch (Exception ex) { - ex.printStackTrace(); + } catch (TemplateException ex) { + throw new IOException("Failed to generate preview.", ex); + } + } + @Override + public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { + try { + //prepare metadata for template + Map dataModel = new HashMap<>(); + dataModel.put("crateModel", mapFromJson(metadata)); + + //prepare output folder and writer + FileUtils.forceMkdir(new File("temp")); + FileWriter writer = new FileWriter(new File("temp/ro-crate-preview.html")); + + //load and process template + Template temp = cfg.getTemplate("templates/custom_preview.frm"); + temp.process(dataModel, writer); + writer.flush(); + writer.close(); + + ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + } catch (TemplateException ex) { + throw new IOException("Failed to generate preview.", ex); + } finally { + FileUtils.deleteDirectory(new File("temp")); } + } + } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java deleted file mode 100644 index e0ee48b6..00000000 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/DefaultPreviewGenerator.java +++ /dev/null @@ -1,143 +0,0 @@ -package edu.kit.datamanager.ro_crate.preview; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel; -import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel.Dataset; -import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel.Part; -import edu.kit.datamanager.ro_crate.preview.model.ROCratePreviewModel.ROCrate; -import edu.kit.datamanager.ro_crate.util.JsonLdExpander; -import freemarker.template.Configuration; -import freemarker.template.Template; -import freemarker.template.TemplateExceptionHandler; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import net.lingala.zip4j.ZipFile; - -/** - * - * @author jejkal - */ -public class DefaultPreviewGenerator { - - public static ROCratePreviewModel mapFromJson(String metadata) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - JsonNode root = (JsonNode) mapper.readValue(metadata, JsonNode.class); - JsonNode graph = root.get("@graph"); - ROCrate crate = new ROCrate(); - List datasets = new ArrayList<>(); - List files = new ArrayList<>(); - - if (graph.isArray()) { - - for (JsonNode node : graph) { - String id = node.get("@id").asText(); - List types = new LinkedList<>(); - if (node.get("@type").isArray()) { - - Collections.addAll(types, (String[]) mapper.convertValue(node.get("@type"), String[].class)); - } else { - types.add(node.get("@type").asText()); - } - - if (types.contains("Dataset") && "./".equals(id)) { - crate.name = node.get("name").asText(); - crate.description = node.get("description") == null ? null : node.get("description").asText(); - crate.type = "Dataset"; - crate.license = node.get("license").isObject() ? node.get("license").get("@id").asText() : node.get("license").asText(); - crate.datePublished = node.get("datePublished") == null ? null : node.get("datePublished").asText(); - crate.hasPart = new ArrayList<>(); - - if (node.has("hasPart")) { - for (JsonNode part : node.get("hasPart")) { - Part p = new Part(); - p.id = part.get("@id").asText(); - p.name = part.get("@id").asText(); // Name will be replaced later - crate.hasPart.add(p); - } - - } - } else if (types.contains("Dataset")) { - Dataset dataset = new Dataset(); - dataset.id = id; - dataset.name = node.get("name").asText(); - dataset.description = node.get("description").asText(); - datasets.add(dataset); - } else if (types.contains("File")) { - ROCratePreviewModel.File file = new ROCratePreviewModel.File(); - file.id = id; - file.name = node.get("name") == null ? null : node.get("name").asText(); - file.description = node.get("description") == null ? null : node.get("description").asText(); - file.contentSize = node.get("contentSize") == null ? null : node.get("contentSize").asText(); - file.encodingFormat = node.get("encodingFormat") == null ? null : node.get("encodingFormat").asText(); - files.add(file); - } - } - } - - // Update Part names using dataset and file lists - if (crate.hasPart != null) { - for (Part part : crate.hasPart) { - for (Dataset dataset : datasets) { - if (dataset.id.equals(part.id) && dataset.name != null) { - part.name = dataset.name; - } - } - for (ROCratePreviewModel.File file : files) { - if (file.id.equals(part.id) && file.name != null) { - part.name = file.name; - } - } - } - } - - ROCratePreviewModel model = new ROCratePreviewModel(); - model.crate = crate; - model.datasets = datasets; - model.files = files; - return model; - } - - public static void generatePreview(String metadata, ZipFile zip) throws IOException { - /* ROCrateModel model = mapFromJson(metadata); - - Map input = new HashMap(); - input.put("model", model); - - CodeResolver codeResolver = new ResourceCodeResolver("templates"); // This is the directory where your .jte files are located. - TemplateEngine templateEngine = TemplateEngine.create(codeResolver, ContentType.Html); - TemplateOutput output = new StringOutput(); - templateEngine.render("default_preview.jte", input, output); - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-preview.html"); - zip.addStream(new ByteArrayInputStream(output.toString().getBytes()), zipParameters);*/ - } - - public static void generatePreview(String metadata, java.io.File folder) throws IOException { - Configuration cfg = new Configuration(Configuration.VERSION_2_3_34); - cfg.setClassForTemplateLoading(DefaultPreviewGenerator.class, "/"); - cfg.setDefaultEncoding("UTF-8"); - cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); - - try { - Map dataModel = new HashMap<>(); - dataModel.put("crateModel", mapFromJson(metadata)); - Template temp = cfg.getTemplate("templates/custom_preview.frm"); - Writer out = new OutputStreamWriter(new FileOutputStream("prev.html")); - - temp.process(dataModel, out); - } catch (Exception ex) { - ex.printStackTrace(); - - } - } - -} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java new file mode 100644 index 00000000..150472cc --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -0,0 +1,70 @@ +package edu.kit.datamanager.ro_crate.preview; + +import edu.kit.datamanager.ro_crate.util.ZipUtil; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import net.lingala.zip4j.model.ZipParameters; +import org.apache.commons.io.FileUtils; + +/** + * This class adds a static preview to the crate, which consists of a + * metadataHtml file and a folder containing other files required to render + * metadataHtml. If will be put unchanged to the writer output, i.e., a zip + * file, folder, or stream. + * + * @author jejkal + */ +public class StaticPreview implements CratePreview { + + private final File metadataHtml; + private final File otherFiles; + + public StaticPreview(File metadataHtml, File otherFiles) { + this.metadataHtml = metadataHtml; + this.otherFiles = otherFiles; + } + + public StaticPreview(File metadataHtml) { + this.metadataHtml = metadataHtml; + this.otherFiles = null; + } + + @Override + public void saveAllToZip(ZipFile zipFile) throws IOException { + if (this.metadataHtml != null) { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-preview.html"); + zipFile.addFile(this.metadataHtml, zipParameters); + } + + if (this.otherFiles != null) { + zipFile.addFolder(this.otherFiles); + zipFile.renameFile(this.otherFiles.getName() + "/", "ro-crate-preview_files/"); + } + } + + @Override + public void saveAllToFolder(File folder) throws IOException { + if (this.metadataHtml != null) { + File fileInCrate = folder.toPath().resolve("ro-crate-preview.html").toFile(); + FileUtils.copyFile(this.metadataHtml, fileInCrate); + } + if (this.otherFiles != null) { + File folderName = folder.toPath().resolve("ro-crate-preview_files").toFile(); + FileUtils.copyDirectory(this.otherFiles, folderName); + } + } + + @Override + public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { + ZipUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); + ZipUtil.addFolderToZipStream(stream, this.otherFiles, this.otherFiles.getName()); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java new file mode 100644 index 00000000..36841318 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java @@ -0,0 +1,55 @@ +package edu.kit.datamanager.ro_crate.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import net.lingala.zip4j.model.ZipParameters; + +/** + * + * @author jejkal + */ +public class ZipUtil { + + public static void addFolderToZipStream(ZipOutputStream zipOutputStream, File folder, String parentPath) throws IOException { + if (!folder.exists() || !folder.isDirectory()) { + throw new IllegalArgumentException("The provided folder path is not a valid directory: " + folder.getAbsolutePath()); + } + + File[] files = folder.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + String zipEntryPath = parentPath.isEmpty() ? file.getName() : parentPath + "/" + file.getName(); + if (file.isDirectory()) { + addFolderToZipStream(zipOutputStream, file.getAbsolutePath(), zipEntryPath); + } else { + addFileToZipStream(zipOutputStream, file, zipEntryPath); + } + } + } + + public static void addFolderToZipStream(ZipOutputStream zipOutputStream, String folderPath, String parentPath) throws IOException { + addFolderToZipStream(zipOutputStream, new File(folderPath), parentPath); + } + + public static void addFileToZipStream(ZipOutputStream zipOutputStream, File file, String zipEntryPath) throws IOException { + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip(zipEntryPath); + zipOutputStream.putNextEntry(zipParameters); + + try (InputStream inputStream = new FileInputStream(file)) { + byte[] buffer = new byte[4096]; + int len; + while ((len = inputStream.read(buffer)) != -1) { + zipOutputStream.write(buffer, 0, len); + } + } + + zipOutputStream.closeEntry(); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java index a3b9e72a..3145c3a1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java @@ -27,7 +27,6 @@ public class ZipStreamWriter implements StreamWriterStrategy { @Override public void save(Crate crate, OutputStream destination) { - try (ZipOutputStream zipFile = new ZipOutputStream(destination)) { saveMetadataJson(crate, zipFile); saveDataEntities(crate, zipFile); @@ -68,11 +67,9 @@ private void saveMetadataJson(Crate crate, ZipOutputStream zipStream) { } zipStream.closeEntry(); - //TODO: Preview not written as creation requires extracted crate, - //which is not possible if directly written to stream. To be done later. - /* if (crate.getPreview() != null) { - crate.getPreview().saveAllToStream(zipStream); - }*/ + if (crate.getPreview() != null) { + crate.getPreview().saveAllToStream(str, zipStream); + } } catch (IOException e) { logger.error("Exception writing ro-crate-metadata.json file to zip.", e); } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java index 67d8024a..dff25fd0 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java @@ -1,6 +1,5 @@ package edu.kit.datamanager.ro_crate.writer; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java index 8fcfd9b8..2f2e308c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java @@ -4,6 +4,7 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.preview.CustomPreview; +import edu.kit.datamanager.ro_crate.preview.StaticPreview; import edu.kit.datamanager.ro_crate.reader.FolderReader; import edu.kit.datamanager.ro_crate.reader.RoCrateReader; import edu.kit.datamanager.ro_crate.writer.FolderWriter; @@ -31,7 +32,7 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { FileUtils.writeStringToFile(fileInDir.toFile(), "fileN2", Charset.defaultCharset()); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .setPreview(new CustomPreview(htmlFile.toFile(), htmlDir.toFile())) + .setPreview(new StaticPreview(htmlFile.toFile(), htmlDir.toFile())) .build(); Path writeDir = path.resolve("crate"); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index a01a915d..e1daf339 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -7,6 +7,7 @@ import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; import edu.kit.datamanager.ro_crate.preview.CustomPreview; +import edu.kit.datamanager.ro_crate.preview.StaticPreview; import edu.kit.datamanager.ro_crate.writer.FolderWriter; import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; @@ -31,7 +32,7 @@ void testAutomaticPreview(@TempDir Path temp) { writer.save(crate, "."); assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); } - + @Test void testAutomaticPreviewAddingLater(@TempDir Path temp) { Path location = temp.resolve("ro_crate2"); @@ -44,13 +45,27 @@ void testAutomaticPreviewAddingLater(@TempDir Path temp) { assertTrue(location.resolve("ro-crate-preview.html").toFile().exists()); } + + @Test + void testCustomPreview(@TempDir Path temp) { + Path location = temp.resolve("ro_crate1"); + RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .setPreview(new CustomPreview()) + .build(); + RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); + writer.save(crate, location.toFile().getAbsolutePath()); + writer.save(crate, "."); + assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); + } + + @Test - void testCustomPreviewOnlyHtmlFile(@TempDir Path temp) throws IOException { + void testStaticPreviewOnlyHtmlFile(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate3"); Path previewFile = temp.resolve("random.html"); FileUtils.writeStringToFile(previewFile.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .setPreview(new CustomPreview(previewFile.toFile())) + .setPreview(new StaticPreview(previewFile.toFile())) .build(); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); writer.save(crate, location.toFile().toString()); @@ -58,7 +73,7 @@ void testCustomPreviewOnlyHtmlFile(@TempDir Path temp) throws IOException { } @Test - void testCustomPreviewHtmlFileWithOtherFiles(@TempDir Path temp) throws IOException { + void testStaticPreviewHtmlFileWithOtherFiles(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate4"); Path previewFile = temp.resolve("random.html"); FileUtils.writeStringToFile(previewFile.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); @@ -66,7 +81,7 @@ void testCustomPreviewHtmlFileWithOtherFiles(@TempDir Path temp) throws IOExcept Path css_file = dirHtml.resolve("test.css"); FileUtils.writeStringToFile(css_file.toFile(), "random css it is not important that it is valid for know", Charset.defaultCharset()); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .setPreview(new CustomPreview(previewFile.toFile(), dirHtml.toFile())) + .setPreview(new StaticPreview(previewFile.toFile(), dirHtml.toFile())) .build(); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); writer.save(crate, location.toFile().toString()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java index e8c7d2ac..545d906d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java @@ -18,14 +18,14 @@ public class PreviewTest { @Test - void customPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { + void staticPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { var file1 = dir.resolve("file.html"); FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); var file2 = dir.resolve("directory"); var fileInDir = file2.resolve("fileInDir.html"); FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); - CustomPreview customPreview = new CustomPreview(file1.toFile(), file2.toFile()); + StaticPreview customPreview = new StaticPreview(file1.toFile(), file2.toFile()); customPreview.saveAllToFolder(dir.resolve("result").toFile()); @@ -45,14 +45,14 @@ void customPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { } @Test - void customPreviewSaveToZip(@TempDir Path dir) throws IOException { + void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { var file1 = dir.resolve("file.html"); FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); var file2 = dir.resolve("directory"); var fileInDir = file2.resolve("fileInDir.html"); FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); - CustomPreview customPreview = new CustomPreview(file1.toFile(), file2.toFile()); + StaticPreview customPreview = new StaticPreview(file1.toFile(), file2.toFile()); customPreview.saveAllToZip(new ZipFile(dir.resolve("destination.zip").toFile())); try (ZipFile zf = new ZipFile(dir.resolve("destination.zip").toFile())) { @@ -71,13 +71,13 @@ void customPreviewSaveToZip(@TempDir Path dir) throws IOException { assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); - } @Test void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); // this crate will not have a json file @@ -115,4 +115,47 @@ void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { zipFile.extractAll(crate.toString()); assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } + + @Test + void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { + CustomPreview customPreview = new CustomPreview(); + + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + Path crate = dir.resolve("crate"); + // this crate will not have a json file + Path fakeCrate = dir.resolve("fakeCrate"); + FileUtils.forceMkdir(crate.toFile()); + FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + + customPreview.saveAllToFolder(crate.toFile()); + + // the program should not crash + customPreview.saveAllToFolder(fakeCrate.toFile()); + + // there should be a html file generated + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } + + @Test + void testCustomPreviewZip(@TempDir Path dir) throws IOException { + CustomPreview customPreview = new CustomPreview(); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + Path crate = dir.resolve("crate"); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + + ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); + zipFile.addStream(crateJson, zipParameters); + crateJson.close(); + + customPreview.saveAllToZip(zipFile); + + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); + customPreview.saveAllToZip(randomZipFile); + + zipFile.extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } + } From f83386a824d0b600fd0f270c80d723d1d3cbd2cc Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Wed, 9 Apr 2025 15:27:33 +0200 Subject: [PATCH 12/88] Fixed all test issues, added some checks to preview generators for more fault tolerance --- .../ro_crate/preview/AutomaticPreview.java | 57 ++- .../ro_crate/preview/CustomPreview.java | 24 +- .../ro_crate/preview/PreviewGenerator.java | 2 - .../ro_crate/preview/StaticPreview.java | 9 +- .../ro_crate/preview/PreviewTest.java | 269 +++++++------ .../ro_crate/writer/FolderWriterTest.java | 370 +++++++++--------- .../ro_crate/writer/ZipWriterTest.java | 287 +++++++------- 7 files changed, 529 insertions(+), 489 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index c840152f..96a36429 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -5,6 +5,7 @@ import java.io.FileWriter; import java.io.IOException; import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; @@ -23,39 +24,57 @@ public AutomaticPreview() { @Override public void saveAllToZip(ZipFile zipFile) throws IOException { - // extract the .json file so we can run the "rochtml" tool on it" - try { - zipFile.extractFile("ro-crate-metadata.json", "temp"); - if (PreviewGenerator.isRochtmlAvailable()) { - PreviewGenerator.generatePreview("temp"); - zipFile.addFile("temp/ro-crate-preview.html"); + if (PreviewGenerator.isRochtmlAvailable()) { + // extract the .json file so we can run the "rochtml" tool on it" + try { + try { + zipFile.extractFile("ro-crate-metadata.json", "temp"); + } catch (ZipException ex) { + throw new IOException("ro-crate-metadata.json not found in provided ZIP.", ex); + } + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview("temp"); + zipFile.addFile("temp/ro-crate-preview.html"); + } + } finally { + FileUtils.deleteDirectory(new File("temp")); } - } finally { - FileUtils.deleteDirectory(new File("temp")); + } else { + new CustomPreview().saveAllToZip(zipFile); } } @Override public void saveAllToFolder(File folder) throws IOException { + if (folder == null || !folder.exists()) { + throw new IOException("Preview target folder " + folder + " does not exist."); + } + if (PreviewGenerator.isRochtmlAvailable()) { PreviewGenerator.generatePreview(folder.getAbsolutePath()); + } else { + new CustomPreview().saveAllToFolder(folder); } } @Override public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { - try { - FileUtils.forceMkdir(new File("temp")); - FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json")); - writer.write(metadata); - writer.flush(); - writer.close(); - if (PreviewGenerator.isRochtmlAvailable()) { - PreviewGenerator.generatePreview("temp"); - ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + if (PreviewGenerator.isRochtmlAvailable()) { + try { + FileUtils.forceMkdir(new File("temp")); + FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json")); + writer.write(metadata); + writer.flush(); + writer.close(); + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview("temp"); + ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); + } + } finally { + FileUtils.deleteDirectory(new File("temp")); } - } finally { - FileUtils.deleteDirectory(new File("temp")); + } else { + new CustomPreview().saveAllToStream(metadata, stream); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index 433de692..e589768a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import org.apache.commons.io.FileUtils; @@ -67,7 +68,9 @@ public ROCratePreviewModel mapFromJson(String metadata) throws IOException { crate.name = node.get("name").asText(); crate.description = node.get("description") == null ? null : node.get("description").asText(); crate.type = "Dataset"; - crate.license = node.get("license").isObject() ? node.get("license").get("@id").asText() : node.get("license").asText(); + if (node.get("license") != null) { + crate.license = node.get("license").isObject() ? node.get("license").get("@id").asText() : node.get("license").asText(); + } crate.datePublished = node.get("datePublished") == null ? null : node.get("datePublished").asText(); crate.hasPart = new ArrayList<>(); @@ -123,7 +126,15 @@ public ROCratePreviewModel mapFromJson(String metadata) throws IOException { @Override public void saveAllToZip(ZipFile zipFile) throws IOException { - zipFile.extractFile("ro-crate-metadata.json", "temp"); + if (zipFile == null) { + throw new IOException("Argument zipFile must not be null."); + } + try { + zipFile.extractFile("ro-crate-metadata.json", "temp"); + } catch (ZipException ex) { + throw new IOException("ro-crate-metadata.json not found in provided ZIP.", ex); + } + String metadata = FileUtils.readFileToString(new File("temp/ro-crate-metadata.json"), "UTF-8"); try { Map dataModel = new HashMap<>(); @@ -145,12 +156,15 @@ public void saveAllToZip(ZipFile zipFile) throws IOException { @Override public void saveAllToFolder(File folder) throws IOException { + if (folder == null || !folder.exists()) { + throw new IOException("Preview target folder " + folder + " does not exist."); + } String metadata = FileUtils.readFileToString(new File(folder, "ro-crate-metadata.json"), "UTF-8"); try { Map dataModel = new HashMap<>(); dataModel.put("crateModel", mapFromJson(metadata)); Template temp = cfg.getTemplate("templates/custom_preview.ftl"); - Writer out = new OutputStreamWriter(new FileOutputStream("prev.html")); + Writer out = new OutputStreamWriter(new FileOutputStream(new File(folder, "ro-crate-preview.html"))); temp.process(dataModel, out); } catch (TemplateException ex) { @@ -164,7 +178,7 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx //prepare metadata for template Map dataModel = new HashMap<>(); dataModel.put("crateModel", mapFromJson(metadata)); - + //prepare output folder and writer FileUtils.forceMkdir(new File("temp")); FileWriter writer = new FileWriter(new File("temp/ro-crate-preview.html")); @@ -174,7 +188,7 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx temp.process(dataModel, writer); writer.flush(); writer.close(); - + ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } catch (TemplateException ex) { throw new IOException("Failed to generate preview.", ex); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java index d170341c..9ba65520 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/PreviewGenerator.java @@ -25,14 +25,12 @@ public static boolean isRochtmlAvailable() { } else { builder.command("sh", "-c", command); } - Process process; try { process = builder.start(); int exitVal = process.waitFor(); return exitVal == 0; } catch (InterruptedException | IOException ex) { - } return false; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index 150472cc..718cb2b0 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -1,14 +1,9 @@ package edu.kit.datamanager.ro_crate.preview; import edu.kit.datamanager.ro_crate.util.ZipUtil; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; import org.apache.commons.io.FileUtils; @@ -52,6 +47,10 @@ public void saveAllToZip(ZipFile zipFile) throws IOException { @Override public void saveAllToFolder(File folder) throws IOException { + if (folder == null || !folder.exists()) { + throw new IOException("Preview target folder " + folder + " does not exist."); + } + if (this.metadataHtml != null) { File fileInCrate = folder.toPath().resolve("ro-crate-preview.html").toFile(); FileUtils.copyFile(this.metadataHtml, fileInCrate); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java index 545d906d..2ef45737 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java @@ -11,151 +11,160 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; +import org.junit.jupiter.api.Assertions; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class PreviewTest { - @Test - void staticPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { - var file1 = dir.resolve("file.html"); - FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + @Test + void staticPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { + var file1 = dir.resolve("file.html"); + FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); - var file2 = dir.resolve("directory"); - var fileInDir = file2.resolve("fileInDir.html"); - FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); - StaticPreview customPreview = new StaticPreview(file1.toFile(), file2.toFile()); + var file2 = dir.resolve("directory"); + var fileInDir = file2.resolve("fileInDir.html"); + FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); + StaticPreview customPreview = new StaticPreview(file1.toFile(), file2.toFile()); - customPreview.saveAllToFolder(dir.resolve("result").toFile()); + FileUtils.forceMkdir(dir.resolve("result").toFile()); + customPreview.saveAllToFolder(dir.resolve("result").toFile()); - var e = dir.resolve("result"); + var e = dir.resolve("result"); - var roPreview = e.resolve("ro-crate-preview.html"); - var roDir = e.resolve("ro-crate-preview_files"); - var roDirFile = roDir.resolve("fileInDir.html"); - assertTrue(Files.isRegularFile(roPreview)); - assertTrue(Files.isDirectory(roDir)); - assertTrue(Files.isRegularFile(roDirFile)); + var roPreview = e.resolve("ro-crate-preview.html"); + var roDir = e.resolve("ro-crate-preview_files"); + var roDirFile = roDir.resolve("fileInDir.html"); + assertTrue(Files.isRegularFile(roPreview)); + assertTrue(Files.isDirectory(roDir)); + assertTrue(Files.isRegularFile(roDirFile)); - assertTrue(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), file1.toFile(), String.valueOf(Charset.defaultCharset()))); - assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + assertTrue(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), file1.toFile(), String.valueOf(Charset.defaultCharset()))); + assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); - assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); - } + assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + } + + @Test + void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { + var file1 = dir.resolve("file.html"); + FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + + var file2 = dir.resolve("directory"); + var fileInDir = file2.resolve("fileInDir.html"); + FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); + StaticPreview customPreview = new StaticPreview(file1.toFile(), file2.toFile()); + + customPreview.saveAllToZip(new ZipFile(dir.resolve("destination.zip").toFile())); + try (ZipFile zf = new ZipFile(dir.resolve("destination.zip").toFile())) { + zf.extractAll(dir.resolve("extracted").toAbsolutePath().toString()); + } + + var e = dir.resolve("extracted"); + var roPreview = e.resolve("ro-crate-preview.html"); + var roDir = e.resolve("ro-crate-preview_files"); + var roDirFile = roDir.resolve("fileInDir.html"); + assertTrue(Files.isRegularFile(roPreview)); + assertTrue(Files.isDirectory(roDir)); + assertTrue(Files.isRegularFile(roDirFile)); + + assertTrue(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), file1.toFile(), String.valueOf(Charset.defaultCharset()))); + assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + + assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); + } - @Test - void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { - var file1 = dir.resolve("file.html"); - FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + @Test + void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { + AutomaticPreview automaticPreview = new AutomaticPreview(); - var file2 = dir.resolve("directory"); - var fileInDir = file2.resolve("fileInDir.html"); - FileUtils.writeStringToFile(fileInDir.toFile(), "dajkdlfjdsklafj alksfjdalk fjl", Charset.defaultCharset()); - StaticPreview customPreview = new StaticPreview(file1.toFile(), file2.toFile()); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + Path crate = dir.resolve("crate"); + // this crate will not have a json file + FileUtils.forceMkdir(crate.toFile()); + FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + automaticPreview.saveAllToFolder(crate.toFile()); + + // there should be a html file generated + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } + + @Test + void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { + AutomaticPreview automaticPreview = new AutomaticPreview(); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + Path crate = dir.resolve("crate"); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + + ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); + zipFile.addStream(crateJson, zipParameters); + crateJson.close(); + + automaticPreview.saveAllToZip(zipFile); + + try { + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); + automaticPreview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid ZIP file for preview."); + } catch (IOException ex) { + //ok + } + zipFile.extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } + + @Test + void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { + CustomPreview customPreview = new CustomPreview(); + + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + Path crate = dir.resolve("crate"); + // this crate will not have a json file + Path fakeCrate = dir.resolve("fakeCrate"); + FileUtils.forceMkdir(crate.toFile()); + FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); + + customPreview.saveAllToFolder(crate.toFile()); + + try { + // this should trow an exception but not stop the execution + customPreview.saveAllToFolder(fakeCrate.toFile()); + Assertions.fail("Expected IOException when providing invalid ZIP file for preview."); + } catch (IOException ex) { + //ok + } + + // there should be a html file generated + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); + } - customPreview.saveAllToZip(new ZipFile(dir.resolve("destination.zip").toFile())); - try (ZipFile zf = new ZipFile(dir.resolve("destination.zip").toFile())) { - zf.extractAll(dir.resolve("extracted").toAbsolutePath().toString()); + @Test + void testCustomPreviewZip(@TempDir Path dir) throws IOException { + CustomPreview customPreview = new CustomPreview(); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + Path crate = dir.resolve("crate"); + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + + ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); + zipFile.addStream(crateJson, zipParameters); + crateJson.close(); + + customPreview.saveAllToZip(zipFile); + + try { + // this should trow an exception but not stop the execution + ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); + customPreview.saveAllToZip(randomZipFile); + Assertions.fail("Expected IOException when providing invalid input to preview."); + } catch (IOException ex) { + //ok + } + zipFile.extractAll(crate.toString()); + assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); } - var e = dir.resolve("extracted"); - var roPreview = e.resolve("ro-crate-preview.html"); - var roDir = e.resolve("ro-crate-preview_files"); - var roDirFile = roDir.resolve("fileInDir.html"); - assertTrue(Files.isRegularFile(roPreview)); - assertTrue(Files.isDirectory(roDir)); - assertTrue(Files.isRegularFile(roDirFile)); - - assertTrue(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), file1.toFile(), String.valueOf(Charset.defaultCharset()))); - assertFalse(FileUtils.contentEqualsIgnoreEOL(roPreview.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); - - assertTrue(FileUtils.contentEqualsIgnoreEOL(roDirFile.toFile(), fileInDir.toFile(), String.valueOf(Charset.defaultCharset()))); - } - - @Test - void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { - AutomaticPreview automaticPreview = new AutomaticPreview(); - - - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); - Path crate = dir.resolve("crate"); - // this crate will not have a json file - Path fakeCrate = dir.resolve("fakeCrate"); - FileUtils.forceMkdir(crate.toFile()); - FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); - - automaticPreview.saveAllToFolder(crate.toFile()); - - // the program should not crash - automaticPreview.saveAllToFolder(fakeCrate.toFile()); - - // there should be a html file generated - assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); - } - - @Test - void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { - AutomaticPreview automaticPreview = new AutomaticPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); - Path crate = dir.resolve("crate"); - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - - ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); - zipFile.addStream(crateJson, zipParameters); - crateJson.close(); - - automaticPreview.saveAllToZip(zipFile); - - // this should trow an exception but not stop the execution - ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); - automaticPreview.saveAllToZip(randomZipFile); - - zipFile.extractAll(crate.toString()); - assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); - } - - @Test - void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { - CustomPreview customPreview = new CustomPreview(); - - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); - Path crate = dir.resolve("crate"); - // this crate will not have a json file - Path fakeCrate = dir.resolve("fakeCrate"); - FileUtils.forceMkdir(crate.toFile()); - FileUtils.copyInputStreamToFile(crateJson, crate.resolve("ro-crate-metadata.json").toFile()); - - customPreview.saveAllToFolder(crate.toFile()); - - // the program should not crash - customPreview.saveAllToFolder(fakeCrate.toFile()); - - // there should be a html file generated - assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); - } - - @Test - void testCustomPreviewZip(@TempDir Path dir) throws IOException { - CustomPreview customPreview = new CustomPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); - Path crate = dir.resolve("crate"); - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - - ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); - zipFile.addStream(crateJson, zipParameters); - crateJson.close(); - - customPreview.saveAllToZip(zipFile); - - // this should trow an exception but not stop the execution - ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); - customPreview.saveAllToZip(randomZipFile); - - zipFile.extractAll(crate.toString()); - assertTrue(Files.isRegularFile(crate.resolve("ro-crate-preview.html"))); - } - } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java index 7f2333f1..ba6ecc1a 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java @@ -14,6 +14,7 @@ import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.CustomPreview; import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; import org.apache.commons.io.FileUtils; @@ -25,188 +26,189 @@ * @version 1 */ class FolderWriterTest { - - - @Test - void writeToFolderCorrectNames(@TempDir Path tempDir) throws IOException { - Path fileWithoutID = tempDir.resolve("spo.txt"); - FileUtils.writeStringToFile(fileWithoutID.toFile(), "content", Charset.defaultCharset()); - Path file1 = tempDir.resolve("input.txt"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = tempDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - Path result = tempDir.resolve("dest"); - - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - folderRoCrateWriter.save(roCrate, result.toFile().toString()); - // test if the names of the files in the crate are correct, - // when there is an ID the file should be called the same as the entity. - assertTrue(Files.isRegularFile(result.resolve("cp7glop.ai"))); - assertTrue(Files.isDirectory(result.resolve("lots_of_little_files/"))); - // assertTrue(Files.isRegularFile(result.resolve(fileWithoutID.getFileName()))); - } - - @Test - void writeToFolderTest(@TempDir Path tempDir) throws IOException { - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson = - FolderWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toString()); - - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("lots_of_little_files"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - Path result = tempDir.resolve("dest"); - folderRoCrateWriter.save(roCrate, result.toFile().toString()); - assertTrue(HelpFunctions.compareTwoDir(result.toFile(), roDir.toFile())); - - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } - - @Test - void writeToFolderWrongTest(@TempDir Path tempDir) throws IOException { - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson = - FolderWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toString()); - - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("lots_of_little_files"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - // false file, this test case should fal - Path falseFile = tempDir.resolve("new"); - FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(falseFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .setId("lots_of_little_files/") - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .build(); - - Path result = tempDir.resolve("dest"); - folderRoCrateWriter.save(roCrate, result.toFile().toString()); - assertFalse(HelpFunctions.compareTwoDir(result.toFile(), roDir.toFile())); - - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } + + @Test + void writeToFolderCorrectNames(@TempDir Path tempDir) throws IOException { + Path fileWithoutID = tempDir.resolve("spo.txt"); + FileUtils.writeStringToFile(fileWithoutID.toFile(), "content", Charset.defaultCharset()); + Path file1 = tempDir.resolve("input.txt"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = tempDir.resolve("dir"); + FileUtils.forceMkdir(dirInCrate.toFile()); + Path dirInDirInCrate = dirInCrate.resolve("last_dir"); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(file1) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()) + .build(); + + Path result = tempDir.resolve("dest"); + + RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); + folderRoCrateWriter.save(roCrate, result.toFile().toString()); + // test if the names of the files in the crate are correct, + // when there is an ID the file should be called the same as the entity. + assertTrue(Files.isRegularFile(result.resolve("cp7glop.ai"))); + assertTrue(Files.isDirectory(result.resolve("lots_of_little_files/"))); + // assertTrue(Files.isRegularFile(result.resolve(fileWithoutID.getFileName()))); + } + + @Test + void writeToFolderTest(@TempDir Path tempDir) throws IOException { + RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); + Path roDir = tempDir.resolve("ro_dir"); + FileUtils.forceMkdir(roDir.toFile()); + + // the .json of our crate + InputStream fileJson + = FolderWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + + // fill the expected directory with files and dirs + Path json = roDir.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + + //create preview via rochtml or custom as fallback + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview(roDir.toString()); + } else { + new CustomPreview().saveAllToFolder(roDir.toFile()); + } + Path file1 = roDir.resolve("cp7glop.ai"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = roDir.resolve("lots_of_little_files"); + FileUtils.forceMkdir(dirInCrate.toFile()); + Path dirInDirInCrate = dirInCrate.resolve("last_dir"); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(file1) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()) + .build(); + + Path result = tempDir.resolve("dest"); + folderRoCrateWriter.save(roCrate, result.toFile().toString()); + assertTrue(HelpFunctions.compareTwoDir(result.toFile(), roDir.toFile())); + + // just so we know the metadata is still valid + HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + } + + @Test + void writeToFolderWrongTest(@TempDir Path tempDir) throws IOException { + RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); + Path roDir = tempDir.resolve("ro_dir"); + FileUtils.forceMkdir(roDir.toFile()); + + // the .json of our crate + InputStream fileJson + = FolderWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + + // fill the expected directory with files and dirs + Path json = roDir.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + + PreviewGenerator.generatePreview(roDir.toString()); + + Path file1 = roDir.resolve("cp7glop.ai"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = roDir.resolve("lots_of_little_files"); + FileUtils.forceMkdir(dirInCrate.toFile()); + Path dirInDirInCrate = dirInCrate.resolve("last_dir"); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + // false file, this test case should fal + Path falseFile = tempDir.resolve("new"); + FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(falseFile) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .setId("lots_of_little_files/") + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .build(); + + Path result = tempDir.resolve("dest"); + folderRoCrateWriter.save(roCrate, result.toFile().toString()); + assertFalse(HelpFunctions.compareTwoDir(result.toFile(), roDir.toFile())); + + HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index 910e1b1b..a3a2e231 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -13,8 +13,8 @@ import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.CustomPreview; import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; -import java.nio.file.Paths; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; @@ -26,151 +26,150 @@ */ class ZipWriterTest { - @Test - void testWritingToZip(@TempDir Path tempDir) throws IOException { - // create the RO_crate directory in the tempDir - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson= - ZipWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toString()); - - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - // safe the crate in the test.zip file - Path test = tempDir.resolve("test.zip"); - // create a Writer for writing RoCrates to zip - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, test.toString()); + @Test + void testWritingToZip(@TempDir Path tempDir) throws IOException { + // create the RO_crate directory in the tempDir + Path roDir = tempDir.resolve("ro_dir"); + FileUtils.forceMkdir(roDir.toFile()); + + // the .json of our crate + InputStream fileJson + = ZipWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + + // fill the expected directory with files and dirs + Path json = roDir.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + + //create preview via rochtml or custom as fallback + if (PreviewGenerator.isRochtmlAvailable()) { + PreviewGenerator.generatePreview(roDir.toString()); + } else { + new CustomPreview().saveAllToFolder(roDir.toFile()); + } + Path file1 = roDir.resolve("cp7glop.ai"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = roDir.resolve("dir"); + FileUtils.forceMkdir(dirInCrate.toFile()); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(file1) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()) + .build(); + + // safe the crate in the test.zip file + Path test = tempDir.resolve("test.zip"); + // create a Writer for writing RoCrates to zip + RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); + // save the content of the roCrate to the dest zip + roCrateZipWriter.save(roCrate, test.toString()); roCrateZipWriter.save(roCrate, "/Users/jejkal/cra.zip"); - - - Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(test.toFile())) { - zf.extractAll(res.toString()); - } - assertTrue(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); - - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } - - - @Test - void testWritingToZipFail(@TempDir Path tempDir) throws IOException { - // create the RO_crate directory in the tempDir - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson= - ZipWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toFile().getAbsolutePath()); - - Path file1 = roDir.resolve("input.txt"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - // false file, this test case should fal - Path falseFile = tempDir.resolve("new"); - FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(falseFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .build(); - - // safe the crate in the test.zip file - Path test = tempDir.resolve("test.zip"); - // create a Writer for writing RoCrates to zip - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, test.toFile().getAbsolutePath()); - Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(test.toFile())) { - zf.extractAll(res.toFile().getAbsolutePath()); + Path res = tempDir.resolve("dest"); + try (ZipFile zf = new ZipFile(test.toFile())) { + zf.extractAll(res.toString()); + } + assertTrue(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); + + // just so we know the metadata is still valid + HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); } - assertFalse(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } + @Test + void testWritingToZipFail(@TempDir Path tempDir) throws IOException { + // create the RO_crate directory in the tempDir + Path roDir = tempDir.resolve("ro_dir"); + FileUtils.forceMkdir(roDir.toFile()); + + // the .json of our crate + InputStream fileJson + = ZipWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + + // fill the expected directory with files and dirs + Path json = roDir.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + + PreviewGenerator.generatePreview(roDir.toFile().getAbsolutePath()); + + Path file1 = roDir.resolve("input.txt"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = roDir.resolve("dir"); + FileUtils.forceMkdir(dirInCrate.toFile()); + FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + // false file, this test case should fal + Path falseFile = tempDir.resolve("new"); + FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", + "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(falseFile) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .build(); + + // safe the crate in the test.zip file + Path test = tempDir.resolve("test.zip"); + // create a Writer for writing RoCrates to zip + RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); + // save the content of the roCrate to the dest zip + roCrateZipWriter.save(roCrate, test.toFile().getAbsolutePath()); + Path res = tempDir.resolve("dest"); + try (ZipFile zf = new ZipFile(test.toFile())) { + zf.extractAll(res.toFile().getAbsolutePath()); + } + assertFalse(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); + + // just so we know the metadata is still valid + HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + } } From 3098d1381d9568e3f22227d00e974f685204dc53 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Thu, 10 Apr 2025 11:06:43 +0200 Subject: [PATCH 13/88] Fixed issues with stale writers --- ro-crate-metadata.json | 22 +++ ro-crate-preview.html | 130 ++++++++++++++++++ .../ro_crate/preview/AutomaticPreview.java | 20 ++- .../ro_crate/preview/CustomPreview.java | 32 +++-- 4 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 ro-crate-metadata.json create mode 100644 ro-crate-preview.html diff --git a/ro-crate-metadata.json b/ro-crate-metadata.json new file mode 100644 index 00000000..f86459e3 --- /dev/null +++ b/ro-crate-metadata.json @@ -0,0 +1,22 @@ +{ + "@context" : "https://w3id.org/ro/crate/1.1/context", + "@graph" : [ { + "name" : "name", + "description" : "description", + "@id" : "./", + "@type" : "Dataset", + "license" : { + "@id" : "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + }, + "datePublished" : "2024" + }, { + "about" : { + "@id" : "./" + }, + "conformsTo" : { + "@id" : "https://w3id.org/ro/crate/1.1" + }, + "@id" : "ro-crate-metadata.json", + "@type" : "CreativeWork" + } ] +} \ No newline at end of file diff --git a/ro-crate-preview.html b/ro-crate-preview.html new file mode 100644 index 00000000..feeda56c --- /dev/null +++ b/ro-crate-preview.html @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

+

name

+ + +⬇️🏷️ Download all the metadata for name in JSON-LD format + + +
+ + +
+ + + +
+
+
+

name

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
@id./
name [?]name
@typeDataset
description [?]description
datePublished [?]2024
license [?]https://creativecommons.org/licenses/by-nc-sa/3.0/au/
Items that reference this one
about [?]ro-crate-metadata.json
+
+ +
+




+
+ + + + + + + diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java index 96a36429..38241476 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/AutomaticPreview.java @@ -37,7 +37,11 @@ public void saveAllToZip(ZipFile zipFile) throws IOException { zipFile.addFile("temp/ro-crate-preview.html"); } } finally { - FileUtils.deleteDirectory(new File("temp")); + try { + FileUtils.deleteDirectory(new File("temp")); + } catch (IOException ex) { + //ignore + } } } else { new CustomPreview().saveAllToZip(zipFile); @@ -62,16 +66,20 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx if (PreviewGenerator.isRochtmlAvailable()) { try { FileUtils.forceMkdir(new File("temp")); - FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json")); - writer.write(metadata); - writer.flush(); - writer.close(); + try (FileWriter writer = new FileWriter(new File("temp/ro-crate-metadata.json"))) { + writer.write(metadata); + writer.flush(); + } if (PreviewGenerator.isRochtmlAvailable()) { PreviewGenerator.generatePreview("temp"); ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } } finally { - FileUtils.deleteDirectory(new File("temp")); + try { + FileUtils.deleteDirectory(new File("temp")); + } catch (IOException ex) { + //ignore + } } } else { new CustomPreview().saveAllToStream(metadata, stream); diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index e589768a..05364b1a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -140,8 +140,10 @@ public void saveAllToZip(ZipFile zipFile) throws IOException { Map dataModel = new HashMap<>(); dataModel.put("crateModel", mapFromJson(metadata)); Template temp = cfg.getTemplate("templates/custom_preview.ftl"); - Writer out = new OutputStreamWriter(new FileOutputStream("temp/ro-crate-preview.html")); - temp.process(dataModel, out); + try (Writer out = new OutputStreamWriter(new FileOutputStream("temp/ro-crate-preview.html"))) { + temp.process(dataModel, out); + out.flush(); + } zipFile.addFile("temp/ro-crate-preview.html"); } catch (TemplateException ex) { throw new IOException("Failed to generate preview.", ex); @@ -164,9 +166,10 @@ public void saveAllToFolder(File folder) throws IOException { Map dataModel = new HashMap<>(); dataModel.put("crateModel", mapFromJson(metadata)); Template temp = cfg.getTemplate("templates/custom_preview.ftl"); - Writer out = new OutputStreamWriter(new FileOutputStream(new File(folder, "ro-crate-preview.html"))); - - temp.process(dataModel, out); + try (Writer out = new OutputStreamWriter(new FileOutputStream(new File(folder, "ro-crate-preview.html")))) { + temp.process(dataModel, out); + out.flush(); + } } catch (TemplateException ex) { throw new IOException("Failed to generate preview.", ex); } @@ -181,21 +184,24 @@ public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOEx //prepare output folder and writer FileUtils.forceMkdir(new File("temp")); - FileWriter writer = new FileWriter(new File("temp/ro-crate-preview.html")); - //load and process template - Template temp = cfg.getTemplate("templates/custom_preview.frm"); - temp.process(dataModel, writer); - writer.flush(); - writer.close(); + try (FileWriter writer = new FileWriter(new File("temp/ro-crate-preview.html"))) { + //load and process template + Template temp = cfg.getTemplate("templates/custom_preview.frm"); + temp.process(dataModel, writer); + writer.flush(); + } ZipUtil.addFileToZipStream(stream, new File("temp/ro-crate-preview.html"), "ro-crate-preview.html"); } catch (TemplateException ex) { throw new IOException("Failed to generate preview.", ex); } finally { - FileUtils.deleteDirectory(new File("temp")); + try { + FileUtils.deleteDirectory(new File("temp")); + } catch (IOException ex) { + //ignore + } } - } } From 4cd37dba7fd3643948bf522067bcf78778509586 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 10 Apr 2025 09:07:12 +0000 Subject: [PATCH 14/88] chore(deps): update dependency font-awesome to v6 --- ro-crate-preview.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ro-crate-preview.html b/ro-crate-preview.html index feeda56c..be1ca71a 100644 --- a/ro-crate-preview.html +++ b/ro-crate-preview.html @@ -45,7 +45,7 @@ - + - - - - - - -
-
-

-

name

- - -⬇️🏷️ Download all the metadata for name in JSON-LD format - - -
- - -
- - - -
-
-
-

name

- - - - -
- - - - - - - - - - - - - - - - - - - - - - - - -
@id./
name [?]name
@typeDataset
description [?]description
datePublished [?]2024
license [?]https://creativecommons.org/licenses/by-nc-sa/3.0/au/
Items that reference this one
about [?]ro-crate-metadata.json
-
- -
-




-
- - - - - - - diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index b065458b..44d8bd50 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -55,7 +55,6 @@ void testCustomPreview(@TempDir Path temp) { .build(); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); writer.save(crate, location.toFile().getAbsolutePath()); - writer.save(crate, "."); assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); } From 5fdb5f03c6ecb55e3bafef3735f4129a79207cd3 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:23:06 +0200 Subject: [PATCH 37/88] refactor: add null check for otherFiles in saveAllToStream method --- .../edu/kit/datamanager/ro_crate/preview/StaticPreview.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java index 718cb2b0..398615c1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/StaticPreview.java @@ -3,6 +3,8 @@ import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; import java.io.IOException; +import java.util.Optional; + import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.io.outputstream.ZipOutputStream; import net.lingala.zip4j.model.ZipParameters; @@ -64,6 +66,8 @@ public void saveAllToFolder(File folder) throws IOException { @Override public void saveAllToStream(String metadata, ZipOutputStream stream) throws IOException { ZipUtil.addFileToZipStream(stream, this.metadataHtml, "ro-crate-preview.html"); - ZipUtil.addFolderToZipStream(stream, this.otherFiles, this.otherFiles.getName()); + if (this.otherFiles != null) { + ZipUtil.addFolderToZipStream(stream, this.otherFiles, this.otherFiles.getName()); + } } } From 95fbe2532f91965eb78423a3113adc3d4555c57e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:30:27 +0200 Subject: [PATCH 38/88] cleanup: fix typo in test string --- .../ro_crate/crate/preview/PreviewCrateTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index 44d8bd50..eb0e5149 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -62,7 +62,7 @@ void testCustomPreview(@TempDir Path temp) { void testStaticPreviewOnlyHtmlFile(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate3"); Path previewFile = temp.resolve("random.html"); - FileUtils.writeStringToFile(previewFile.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + FileUtils.writeStringToFile(previewFile.toFile(), "random html it is not important that it is valid", Charset.defaultCharset()); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new StaticPreview(previewFile.toFile())) .build(); @@ -75,10 +75,10 @@ void testStaticPreviewOnlyHtmlFile(@TempDir Path temp) throws IOException { void testStaticPreviewHtmlFileWithOtherFiles(@TempDir Path temp) throws IOException { Path location = temp.resolve("ro_crate4"); Path previewFile = temp.resolve("random.html"); - FileUtils.writeStringToFile(previewFile.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + FileUtils.writeStringToFile(previewFile.toFile(), "random html it is not important that it is valid", Charset.defaultCharset()); Path dirHtml = temp.resolve("html_dir"); Path css_file = dirHtml.resolve("test.css"); - FileUtils.writeStringToFile(css_file.toFile(), "random css it is not important that it is valid for know", Charset.defaultCharset()); + FileUtils.writeStringToFile(css_file.toFile(), "random css it is not important that it is valid", Charset.defaultCharset()); RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new StaticPreview(previewFile.toFile(), dirHtml.toFile())) .build(); From 5a9555f9aa1634d2c345516cc1dd50b1f62548ee Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:31:27 +0200 Subject: [PATCH 39/88] fix: correct logger reference in StreamReaderStrategy interface --- .../kit/datamanager/ro_crate/reader/StreamReaderStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java index 810c11b0..390e2788 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java @@ -14,7 +14,7 @@ */ public interface StreamReaderStrategy extends ReaderStrategy { - static org.slf4j.Logger logger = LoggerFactory.getLogger(StreamWriterStrategy.class); + org.slf4j.Logger logger = LoggerFactory.getLogger(StreamReaderStrategy.class); /** * Default override of readMetadataJson interface from ReaderStrategy. The From 578ad33c4ba773930c38c2e81308de4aa01178ed Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:32:38 +0200 Subject: [PATCH 40/88] docs: add interface description for StreamReaderStrategy --- .../kit/datamanager/ro_crate/reader/StreamReaderStrategy.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java index 390e2788..3dfcc243 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory; /** + * Interface for reading RO-Crate metadata and content from input streams. * * @author jejkal */ From c151ad3b777b74679b6d7ff9708958a6793590e8 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:36:54 +0200 Subject: [PATCH 41/88] fix: update error message for StreamReaderStrategy implementation --- .../java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java index 44023d04..961335d3 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java @@ -95,7 +95,7 @@ public RoCrate readCrate(InputStream source) { usedFiles.add(content.toPath().resolve(FILE_PREVIEW_FILES).toFile().getPath()); result = rebuildCrate(metadata, content, usedFiles); } else { - logger.error("Provided writer does not implement StreamWriterStrategy. Please use 'save(Crate crate, String destination)'."); + logger.error("Provided writer does not implement StreamReaderStrategy. Please use 'readCrate(String location)'."); } return result; } From bb4796f17aec3361067b927de4cef5300fa47ce3 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:54:06 +0200 Subject: [PATCH 42/88] feat: add footer to custom preview with attribution message --- src/main/resources/templates/custom_preview.ftl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/resources/templates/custom_preview.ftl b/src/main/resources/templates/custom_preview.ftl index 9d516a62..ae661702 100644 --- a/src/main/resources/templates/custom_preview.ftl +++ b/src/main/resources/templates/custom_preview.ftl @@ -191,5 +191,11 @@ +
+

+ This human-readable preview has been created using ro-crate-java. +

+
+ \ No newline at end of file From 7024885c3159d4c9a58786c2cf8c74e73bcaa7cb Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:54:22 +0200 Subject: [PATCH 43/88] fix: add null checks for metadata JSON and content files in rebuildCrate method --- .../kit/datamanager/ro_crate/reader/RoCrateReader.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java index 961335d3..087888a5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java @@ -122,6 +122,14 @@ public RoCrate readCrate(String location) { } private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet usedFiles) { + if (metadataJson == null) { + logger.error("Metadata JSON is null, cannot rebuild crate"); + return null; + } + if (files == null) { + logger.error("Content files directory is null, cannot rebuild crate"); + return null; + } JsonNode context = metadataJson.get(PROP_CONTEXT); CrateMetadataContext crateContext = new RoCrateMetadataContext(context); From 9779b0b38f3305eec40c6ba6db980e285b34362b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:54:33 +0200 Subject: [PATCH 44/88] fix: remove redundant save call in PreviewCrateTest --- .../kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index eb0e5149..64888f08 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -29,7 +29,6 @@ void testAutomaticPreview(@TempDir Path temp) { .build(); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); writer.save(crate, location.toFile().getAbsolutePath()); - writer.save(crate, "."); assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); } From 6bde4de701f195c5ef5bd79a967d0e73215049a0 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 17:58:29 +0200 Subject: [PATCH 45/88] docs: add class-level documentation for ZipStreamReader --- .../edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java index 125448c7..e2bc274d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java @@ -15,6 +15,9 @@ import org.apache.commons.io.FileUtils; /** + * A ZIP file reader implementation of the StreamReaderStrategy interface. + * This class handles reading and extraction of RO-Crate content from ZIP archives + * into a temporary directory structure, which allows for accessing the contained files. * * @author jejkal */ From 222a0007aef7b4975b853f2b6cc3a63abf977309 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 18:03:18 +0200 Subject: [PATCH 46/88] docs: correct typo in method documentation for readCrateMetadata --- .../edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java index e2bc274d..4f64fa31 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java @@ -76,7 +76,7 @@ public boolean isExtracted() { return isExtracted; } - /**Read the create metadata and content from the provided input stream. + /**Read the crate metadata and content from the provided input stream. * * @param stream The input stream. */ From f76db9d5763413cba25961ebdf5f5dc13b92d6a2 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 18:03:42 +0200 Subject: [PATCH 47/88] fix: validate extracted file paths to prevent directory traversal vulnerabilities --- .../kit/datamanager/ro_crate/reader/ZipStreamReader.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java index 4f64fa31..b71212bb 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java @@ -100,7 +100,11 @@ private void readCrate(InputStream stream) { try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { while ((localFileHeader = zipInputStream.getNextEntry()) != null) { - File extractedFile = new File(folder, localFileHeader.getFileName()); + String fileName = localFileHeader.getFileName(); + File extractedFile = new File(folder, fileName).getCanonicalFile(); + if (!extractedFile.toPath().startsWith(folder.getCanonicalPath())) { + throw new IOException("Entry is outside of target directory: " + fileName); + } try (OutputStream outputStream = new FileOutputStream(extractedFile)) { while ((readLen = zipInputStream.read(readBuffer)) != -1) { outputStream.write(readBuffer, 0, readLen); From 5f16590577709bac9380a97b4b73f4d04f802d89 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 18:06:06 +0200 Subject: [PATCH 48/88] fix: add missing parentheses in README example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a908b96d..e5613a23 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ For StaticPreview, the constuctor is a bit different, such that it looks as foll File pathToMainPreviewHtml = new File("localPath"); File pathToAdditionalFiles = new File("localFolder"); RoCrate roCrate = new RoCrateBuilder("name", "description", "datePublished", "licenseIdentifier") - .setPreview(new StaticPreview(pathToMainPreviewHtml, pathToAdditionalFiles) + .setPreview(new StaticPreview(pathToMainPreviewHtml, pathToAdditionalFiles)) .build(); ``` From 4736e286556f5f4b10220d9de90c92b716382385 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 18:19:27 +0200 Subject: [PATCH 49/88] fix: replace get() with path() for safer JSON node access in CustomPreview --- .../ro_crate/preview/CustomPreview.java | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index f77a4654..4b1349b4 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -63,51 +63,43 @@ private CustomPreviewModel mapFromJson(String metadata) throws IOException { if (graph.isArray()) { for (JsonNode node : graph) { - String id = node.get("@id").asText(); + String id = node.path("@id").asText(); List types = new LinkedList<>(); - if (node.get("@type").isArray()) { + if (node.path("@type").isArray()) { - Collections.addAll(types, (String[]) mapper.convertValue(node.get("@type"), String[].class)); + Collections.addAll(types, mapper.convertValue(node.path("@type"), String[].class)); } else { - types.add(node.get("@type").asText()); + types.add(node.path("@type").asText()); } if (types.contains("Dataset") && "./".equals(id)) { - crate.name = node.get("name").asText(); - crate.description = node.get("description") == null ? null : node.get("description").asText(); + crate.name = node.path("name").asText(); + crate.description = node.path("description").asText(null); crate.type = "Dataset"; - if (node.get("license") != null) { - crate.license = node.get("license").isObject() ? node.get("license").get("@id").asText() : node.get("license").asText(); - } - crate.datePublished = node.get("datePublished") == null ? null : node.get("datePublished").asText(); + crate.license = node.path("license").path("@id").asText(node.path("license").asText(null)); + crate.datePublished = node.path("datePublished").asText(null); crate.hasPart = new ArrayList<>(); - if (node.has("hasPart")) { - for (JsonNode part : node.get("hasPart")) { - CustomPreviewModel.Part p = new CustomPreviewModel.Part(); - if (part.isObject()) { - p.id = part.get("@id").asText(); - p.name = part.get("@id").asText(); // Name will be replaced later - } else { - p.id = part.asText(); - } - - crate.hasPart.add(p); - } + for (JsonNode part : node.path("hasPart")) { + CustomPreviewModel.Part p = new CustomPreviewModel.Part(); + String tmpId = part.path("@id").asText(part.asText()); + p.id = tmpId; + p.name = tmpId; // Name will be replaced later + crate.hasPart.add(p); } } else if (types.contains("Dataset")) { CustomPreviewModel.Dataset dataset = new CustomPreviewModel.Dataset(); dataset.id = id; - dataset.name = node.get("name").asText(); - dataset.description = node.get("description").asText(); + dataset.name = node.path("name").asText(); + dataset.description = node.path("description").asText(); datasets.add(dataset); } else if (types.contains("File")) { CustomPreviewModel.File file = new CustomPreviewModel.File(); file.id = id; - file.name = node.get("name") == null ? null : node.get("name").asText(); - file.description = node.get("description") == null ? null : node.get("description").asText(); - file.contentSize = node.get("contentSize") == null ? null : node.get("contentSize").asText(); - file.encodingFormat = node.get("encodingFormat") == null ? null : node.get("encodingFormat").asText(); + file.name = node.path("name").asText(null); + file.description = node.path("description").asText(null); + file.contentSize = node.path("contentSize").asText(null); + file.encodingFormat = node.path("encodingFormat").asText(null); files.add(file); } } From fe1bb105fe1d94d782dac1cc7ef255a6b72b4679 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 16 Apr 2025 18:37:51 +0200 Subject: [PATCH 50/88] fix: add null and existence checks for JSON-LD file in JsonLdExpander --- .../edu/kit/datamanager/ro_crate/util/JsonLdExpander.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java index 17b7b571..05622a97 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java @@ -9,7 +9,7 @@ /** * Utility class for expanding and pruning JSON-LD documents. - * + *

* This class provides functionality to resolve references in JSON-LD based on "@id" values, * expanding the data structure by replacing references with their full content. * It also handles circular references to prevent infinite recursion. @@ -19,6 +19,12 @@ public class JsonLdExpander { public static JsonNode expandAndPrune(File jsonLdFile) throws Exception { + if (jsonLdFile == null) { + throw new IllegalArgumentException("JSON-LD file must not be null."); + } + if (!jsonLdFile.exists() || !jsonLdFile.canRead()) { + throw new IllegalArgumentException("JSON-LD file does not exist or cannot be read: " + jsonLdFile.getAbsolutePath()); + } ObjectMapper mapper = new ObjectMapper(); JsonNode root = mapper.readTree(jsonLdFile); return expandAndPrune(root); From d5eb38464332e4f0ddd701ac9a9e4e0678bc3956 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 22 Apr 2025 15:44:08 +0200 Subject: [PATCH 51/88] feat: introduce generic readers instead of the StreamReaderStrategy. This has several advantages: - no instanceof - no handling of non-intended use required - only one interface with no restrictions on type T Also provides a helper/factory function for the construction of available readers. Downsides: - Deprecates some existing API for the next major version - Generics required in reading code (e.g. CrateReader in future) --- .../ro_crate/reader/CrateReader.java | 340 +++++++++++++++++ .../ro_crate/reader/FolderReader.java | 33 +- .../ro_crate/reader/FolderStrategy.java | 36 ++ .../reader/GenericReaderStrategy.java | 28 ++ .../ro_crate/reader/ReaderStrategy.java | 14 +- .../datamanager/ro_crate/reader/Readers.java | 77 ++++ .../ro_crate/reader/RoCrateReader.java | 357 +----------------- .../ro_crate/reader/StreamReaderStrategy.java | 64 ---- .../ro_crate/reader/ZipReader.java | 97 +---- .../ro_crate/reader/ZipStrategy.java | 126 +++++++ ...reamReader.java => ZipStreamStrategy.java} | 9 +- ...erTest.java => ZipStreamStrategyTest.java} | 19 +- 12 files changed, 646 insertions(+), 554 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java delete mode 100644 src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java rename src/main/java/edu/kit/datamanager/ro_crate/reader/{ZipStreamReader.java => ZipStreamStrategy.java} (93%) rename src/test/java/edu/kit/datamanager/ro_crate/reader/{ZipStreamReaderTest.java => ZipStreamStrategyTest.java} (92%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java new file mode 100644 index 00000000..35fe0f66 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -0,0 +1,340 @@ +package edu.kit.datamanager.ro_crate.reader; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.context.CrateMetadataContext; +import edu.kit.datamanager.ro_crate.context.RoCrateMetadataContext; +import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; +import edu.kit.datamanager.ro_crate.special.IdentifierUtils; +import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions; +import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; +import edu.kit.datamanager.ro_crate.validation.Validator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * This class allows reading crates from the outside into the library in order + * to inspect or modify it. + *

+ * The constructor takes a strategy to support different ways of importing the + * crates. (from zip, folder, etc.). + *

+ * The reader consideres "hasPart" and "isPartOf" properties and considers all + * entities (in-)directly connected to the root entity ("./") as DataEntities. + * + * @param the type of the location parameter + */ +public class CrateReader { + + private static final Logger logger = LoggerFactory.getLogger(CrateReader.class); + + /** + * This is a private inner class that shall not be exposed. **Do not make it + * public or protected.** It serves only the purpose of unsafe operations + * while reading a crate and may be specific to this implementation. + */ + private static class RoCrateUnsafe extends RoCrate { + + public void addDataEntityWithoutRootHasPart(DataEntity entity) { + this.metadataContext.checkEntity(entity); + this.roCratePayload.addDataEntity(entity); + } + } + + /** + * If the number of JSON entities in the crate is larger than this number, + * parallelization will be used. + */ + private static final int PARALLELIZATION_THRESHOLD = 100; + + private static final String FILE_PREVIEW_FILES = "ro-crate-preview_files"; + private static final String FILE_PREVIEW_HTML = "ro-crate-preview.html"; + private static final String FILE_METADATA_JSON = "ro-crate-metadata.json"; + + protected static final String SPECIFICATION_PREFIX = "https://w3id.org/ro/crate/"; + + protected static final String PROP_ABOUT = "about"; + protected static final String PROP_CONTEXT = "@context"; + protected static final String PROP_CONFORMS_TO = "conformsTo"; + protected static final String PROP_GRAPH = "@graph"; + protected static final String PROP_HAS_PART = "hasPart"; + protected static final String PROP_ID = "@id"; + + private final GenericReaderStrategy strategy; + + public CrateReader(GenericReaderStrategy strategy) { + this.strategy = strategy; + } + + /** + * This function will read the location (using one of the specified + * strategies) and then build the relation between the entities. + * + * @param location the location of the ro-crate to be read + * @return the read RO-crate + */ + public RoCrate readCrate(T location) { + // get the ro-crate-metadata.json + ObjectNode metadataJson = strategy.readMetadataJson(location); + // get the content of the crate + File files = strategy.readContent(location); + + // this set will contain the files that are associated with entities + HashSet usedFiles = new HashSet<>(); + usedFiles.add(files.toPath().resolve(FILE_METADATA_JSON).toFile().getPath()); + usedFiles.add(files.toPath().resolve(FILE_PREVIEW_HTML).toFile().getPath()); + usedFiles.add(files.toPath().resolve(FILE_PREVIEW_FILES).toFile().getPath()); + return rebuildCrate(metadataJson, files, usedFiles); + } + + private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet usedFiles) { + if (metadataJson == null) { + logger.error("Metadata JSON is null, cannot rebuild crate"); + return null; + } + if (files == null) { + logger.error("Content files directory is null, cannot rebuild crate"); + return null; + } + JsonNode context = metadataJson.get(PROP_CONTEXT); + + CrateMetadataContext crateContext = new RoCrateMetadataContext(context); + RoCrateUnsafe crate = new RoCrateUnsafe(); + crate.setMetadataContext(crateContext); + JsonNode graph = metadataJson.get(PROP_GRAPH); + + if (graph.isArray()) { + moveRootEntitiesFromGraphToCrate(crate, (ArrayNode) graph); + RootDataEntity root = crate.getRootDataEntity(); + if (root != null) { + Set dataEntityIds = getDataEntityIds(root, graph); + for (JsonNode entityJson : graph) { + String eId = unpackId(entityJson); + if (dataEntityIds.contains(eId)) { + // data entity + DataEntity.DataEntityBuilder dataEntity = new DataEntity.DataEntityBuilder() + .setAll(entityJson.deepCopy()); + + // Handle data entities with corresponding file + checkFolderHasFile(entityJson.get(PROP_ID).asText(), files).ifPresent(file -> { + usedFiles.add(file.getPath()); + dataEntity.setLocationWithExceptions(file.toPath()) + .setId(file.getName()); + }); + + crate.addDataEntityWithoutRootHasPart(dataEntity.build()); + } else { + // contextual entity + crate.addContextualEntity( + new ContextualEntity.ContextualEntityBuilder() + .setAll(entityJson.deepCopy()) + .build()); + } + } + } + } + + Collection untrackedFiles = Arrays.stream( + Optional.ofNullable(files.listFiles()).orElse(new File[0])) + .filter(f -> !usedFiles.contains(f.getPath())) + .collect(Collectors.toSet()); + + crate.setUntrackedFiles(untrackedFiles); + Validator defaultValidation = new Validator(new JsonSchemaValidation()); + defaultValidation.validate(crate); + return crate; + } + + /** + * Extracts graph connections from top to bottom. + *

+ * Example: (connections.get(parent) -> children) + * + * @param graph the ArrayNode with all Entities. + * @return the graph connections. + */ + protected Map> makeEntityGraph(JsonNode graph) { + Map> connections = new HashMap<>(); + + Map idToNodes = new HashMap<>(); + StreamSupport.stream(graph.spliterator(), false) + .forEach(jsonNode -> idToNodes.put(unpackId(jsonNode), jsonNode)); + + for (JsonNode entityNode : graph) { + String currentId = unpackId(entityNode); + StreamSupport.stream(entityNode.path("hasPart").spliterator(), false) + .map(this::unpackId) + .map(s -> idToNodes.getOrDefault(s, null)) + .filter(Objects::nonNull) + .forEach(child -> connections.computeIfAbsent(currentId, key -> new HashSet<>()) + .add(unpackId(child))); + StreamSupport.stream(entityNode.path("isPartOf").spliterator(), false) + .map(this::unpackId) + .map(s -> idToNodes.getOrDefault(s, null)) + .filter(Objects::nonNull) + .forEach(parent -> connections.computeIfAbsent(unpackId(parent), key -> new HashSet<>()) + .add(currentId)); + } + return connections; + } + + protected Set getDataEntityIds(RootDataEntity root, JsonNode graph) { + if (root == null) { + return Set.of(); + } + Map> network = makeEntityGraph(graph); + Set directDataEntities = new HashSet<>(root.hasPart); + + Stack processingQueue = new Stack<>(); + processingQueue.addAll(directDataEntities); + Set result = new HashSet<>(); + + while (!processingQueue.empty()) { + String currentId = processingQueue.pop(); + result.add(currentId); + network.getOrDefault(currentId, new HashSet<>()).stream() + .filter(subId -> !result.contains(subId)) // avoid loops! + .forEach(subId -> { + result.add(subId); + processingQueue.add(subId); + }); + } + return result; + } + + protected String unpackId(JsonNode node) { + if (node.isTextual()) { + return node.asText(); + } else /*if (node.isObject())*/ { + return node.path(PROP_ID).asText(); + } + } + + protected Optional checkFolderHasFile(String filepathOrId, File folder) { + if (IdentifierUtils.isUrl(filepathOrId)) { + return Optional.empty(); + } + return IdentifierUtils.decode(filepathOrId) + .map(decoded -> folder.toPath().resolve(decoded).toFile()) + .filter(File::exists); + } + + /** + * Moves the descriptor and the root entity from the graph to the crate. + *

+ * Extracts the root data entity and the Metadata File Descriptor from the + * graph and inserts them into the crate object. It also deletes it from the + * graph. We will need the root dataset to distinguish between data entities + * and contextual entities. + * + * @param crate the crate, which will receive the entities, if available in + * the graph. + * @param graph the graph of the Metadata JSON file, where the entities are + * extracted and removed from. + */ + protected void moveRootEntitiesFromGraphToCrate(RoCrate crate, ArrayNode graph) { + Optional maybeDescriptor = getMetadataDescriptor(graph); + + maybeDescriptor.ifPresent(descriptor -> { + setCrateDescriptor(crate, descriptor); + JsonUtilFunctions.removeJsonNodeFromArrayNode(graph, descriptor); + + Optional maybeRoot = extractRoot(graph, descriptor); + + maybeRoot.ifPresent(root -> { + Set hasPartIds = extractHasPartIds(root); + + crate.setRootDataEntity( + new RootDataEntity.RootDataEntityBuilder() + .setAll(root.deepCopy()) + .setHasPart(hasPartIds) + .build()); + + JsonUtilFunctions.removeJsonNodeFromArrayNode(graph, root); + }); + }); + } + + /** + * Find the metadata descriptor. + *

+ * Currently prefers algorithm of version 1.1 over the one of 1.2-DRAFT. + * + * @param graph the graph to search the descriptor in. + * @return the metadata descriptor of the crate. + */ + protected Optional getMetadataDescriptor(ArrayNode graph) { + boolean isParallel = graph.size() > PARALLELIZATION_THRESHOLD; + // use the algorithm described here: + // https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity + Optional maybeDescriptor = StreamSupport.stream(graph.spliterator(), isParallel) + // "2. if the conformsTo property is a URI that starts with + // https://w3id.org/ro/crate/" + .filter(node -> node.path(PROP_CONFORMS_TO).path(PROP_ID).asText().startsWith(SPECIFICATION_PREFIX)) + // "3. from this entity’s about object keep the @id URI as variable root" + .filter(node -> node.path(PROP_ABOUT).path(PROP_ID).isTextual()) + // There should be only one descriptor. If multiple exist, we take the first + // one. + .findFirst(); + return maybeDescriptor.or(() + -> // from https://www.researchobject.org/ro-crate/1.2-DRAFT/root-data-entity.html#finding-the-root-data-entity + StreamSupport.stream(graph.spliterator(), isParallel) + .filter(node -> node.path(PROP_ID).asText().equals(FILE_METADATA_JSON)) + .findFirst() + ); + } + + /** + * Extracts the root entity from the graph, using the information from the + * descriptor. + *

+ * Basically implements step 5 of the algorithm described here: + * + * https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity + * + * + * @param graph the graph from the metadata JSON-LD file + * @param descriptor the RO-Crate descriptor + * @return the root entity, if found + */ + private Optional extractRoot(ArrayNode graph, JsonNode descriptor) { + String rootId = descriptor.get(PROP_ABOUT).get(PROP_ID).asText(); + boolean isParallel = graph.size() > PARALLELIZATION_THRESHOLD; + return StreamSupport.stream(graph.spliterator(), isParallel) + // root is an object (filter + conversion) + .filter(JsonNode::isObject) + .map(JsonNode::deepCopy) + // "5. if the entity has an @id URI that matches root return it" + .filter(node -> node.path(PROP_ID).asText().equals(rootId)) + .findFirst(); + } + + private Set extractHasPartIds(ObjectNode root) { + JsonNode hasPartNode = root.path(PROP_HAS_PART); + boolean isParallel = hasPartNode.isArray() && hasPartNode.size() > PARALLELIZATION_THRESHOLD; + Set hasPartIds = StreamSupport.stream(hasPartNode.spliterator(), isParallel) + .map(hasPart -> hasPart.path(PROP_ID).asText()) + .filter(text -> !text.isBlank()) + .collect(Collectors.toSet()); + if (hasPartIds.isEmpty() && hasPartNode.path(PROP_ID).isTextual()) { + hasPartIds.add(hasPartNode.path(PROP_ID).asText()); + } + return hasPartIds; + } + + private void setCrateDescriptor(RoCrate crate, JsonNode descriptor) { + ContextualEntity descriptorEntity = new ContextualEntity.ContextualEntityBuilder() + .setAll(descriptor.deepCopy()) + .build(); + crate.setJsonDescriptor(descriptorEntity); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java index faaede49..f0436e27 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderReader.java @@ -1,37 +1,12 @@ package edu.kit.datamanager.ro_crate.reader; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; - /** * A class for reading a crate from a folder. * * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 + * + * @deprecated Use {@link FolderStrategy} instead. */ -public class FolderReader implements ReaderStrategy { - - @Override - public ObjectNode readMetadataJson(String location) { - Path metadata = new File(location).toPath().resolve("ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - ObjectNode objectNode = objectMapper.createObjectNode(); - try { - objectNode = objectMapper.readTree(metadata.toFile()).deepCopy(); - } catch (IOException e) { - e.printStackTrace(); - } - return objectNode; - } - - @Override - public File readContent(String location) { - return new File(location); - } -} +@Deprecated(since = "2.1.0", forRemoval = true) +public class FolderReader extends FolderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java new file mode 100644 index 00000000..f6e235ed --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/FolderStrategy.java @@ -0,0 +1,36 @@ +package edu.kit.datamanager.ro_crate.reader; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +/** + * A class for reading a crate from a folder. + * + * @author Nikola Tzotchev on 9.2.2022 г. + * @version 1 + */ +public class FolderStrategy implements GenericReaderStrategy { + + @Override + public ObjectNode readMetadataJson(String location) { + Path metadata = new File(location).toPath().resolve("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + ObjectNode objectNode = objectMapper.createObjectNode(); + try { + objectNode = objectMapper.readTree(metadata.toFile()).deepCopy(); + } catch (IOException e) { + e.printStackTrace(); + } + return objectNode; + } + + @Override + public File readContent(String location) { + return new File(location); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java new file mode 100644 index 00000000..c3539b17 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/GenericReaderStrategy.java @@ -0,0 +1,28 @@ +package edu.kit.datamanager.ro_crate.reader; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.File; + +/** + * Generic interface for the strategy of the reader class. + * This allows for flexible input types when implementing different reading strategies. + * + * @param the type of the location parameter + */ +public interface GenericReaderStrategy { + /** + * Read the metadata.json file from the given location. + * + * @param location the location to read from + * @return the parsed metadata.json as ObjectNode + */ + ObjectNode readMetadataJson(T location); + + /** + * Read the content from the given location. + * + * @param location the location to read from + * @return the content as a File + */ + File readContent(T location); +} \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReaderStrategy.java index 506dfd4f..2029fe9c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ReaderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ReaderStrategy.java @@ -1,18 +1,14 @@ package edu.kit.datamanager.ro_crate.reader; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.io.File; - /** - * Interface for the strategy fo the reader class. + * Interface for the strategy for the reader class. * This should be implemented if additional strategies are to be build. * (e.g., reading from a gzip) * * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 + * + * @deprecated Use {@link GenericReaderStrategy} instead. */ -public interface ReaderStrategy { - ObjectNode readMetadataJson(String location); - - File readContent(String location); -} +@Deprecated(since = "2.1.0", forRemoval = true) +public interface ReaderStrategy extends GenericReaderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java new file mode 100644 index 00000000..49e09cb1 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/Readers.java @@ -0,0 +1,77 @@ +package edu.kit.datamanager.ro_crate.reader; + +import java.io.InputStream; +import java.nio.file.Path; + +/** + * Factory for creating common RO-Crate reader instances. + * Provides convenient static methods to instantiate readers with pre-configured strategies. + */ +public class Readers { + + /** + * Private constructor to prevent instantiation of this utility class. + */ + private Readers() {} + + /** + * Creates a reader that reads from ZIP files using input streams. + * + * @return A reader configured for ZIP files + * + * @see ZipStreamStrategy#ZipStreamStrategy() + */ + public static CrateReader newZipStreamReader() { + return new CrateReader<>(new ZipStreamStrategy()); + } + + /** + * Creates a reader that reads from ZIP files using input streams, + * extracting to a custom temporary location. + * + * @param extractPath Path where ZIP contents should be extracted + * @param useUuidSubfolder Whether to create a UUID subfolder under extractPath + * @return A reader configured for ZIP files with custom extraction + * + * @see ZipStreamStrategy#ZipStreamStrategy(Path, boolean) + */ + public static CrateReader newZipStreamReader(Path extractPath, boolean useUuidSubfolder) { + return new CrateReader<>(new ZipStreamStrategy(extractPath, useUuidSubfolder)); + } + + /** + * Creates a reader that reads from a folder using a string path. + * + * @return A reader configured for folders + * + * @see FolderStrategy + */ + public static CrateReader newFolderReader() { + return new CrateReader<>(new FolderStrategy()); + } + + /** + * Creates a reader that reads from a ZIP file using a string path. + * + * @return A reader configured for ZIP files + * + * @see ZipStrategy#ZipStrategy() + */ + public static CrateReader newZipPathReader() { + return new CrateReader<>(new ZipStrategy()); + } + + /** + * Creates a reader that reads from a ZIP file using a string path, + * extracting to a custom temporary location. + * + * @param extractPath Path where ZIP contents should be extracted + * @param useUuidSubfolder Whether to create a UUID subfolder under extractPath + * @return A reader configured for ZIP files with custom extraction + * + * @see ZipStrategy#ZipStrategy(Path, boolean) + */ + public static CrateReader newZipPathReader(Path extractPath, boolean useUuidSubfolder) { + return new CrateReader<>(new ZipStrategy(extractPath, useUuidSubfolder)); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java index 087888a5..d11177c1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java @@ -1,29 +1,5 @@ package edu.kit.datamanager.ro_crate.reader; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.context.CrateMetadataContext; -import edu.kit.datamanager.ro_crate.context.RoCrateMetadataContext; -import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; -import edu.kit.datamanager.ro_crate.special.IdentifierUtils; -import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions; - -import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; -import edu.kit.datamanager.ro_crate.validation.Validator; - -import java.io.File; -import java.io.InputStream; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * This class allows reading crates from the outside into the library in order * to inspect or modify it. @@ -33,333 +9,12 @@ *

* The reader consideres "hasPart" and "isPartOf" properties and considers all * entities (in-)directly connected to the root entity ("./") as DataEntities. + * + * @deprecated Use {@link CrateReader} instead. */ -public class RoCrateReader { - - private static Logger logger = LoggerFactory.getLogger(RoCrateReader.class); - - /** - * This is a private inner class that shall not be exposed. **Do not make it - * public or protected.** It serves only the purpose of unsafe operations - * while reading a crate and may be specific to this implementation. - */ - private static class RoCrateUnsafe extends RoCrate { - - public void addDataEntityWithoutRootHasPart(DataEntity entity) { - this.metadataContext.checkEntity(entity); - this.roCratePayload.addDataEntity(entity); - } - } - - /** - * If the number of JSON entities in the crate is larger than this number, - * parallelization will be used. - */ - private static final int PARALLELIZATION_THRESHOLD = 100; - - private static final String FILE_PREVIEW_FILES = "ro-crate-preview_files"; - private static final String FILE_PREVIEW_HTML = "ro-crate-preview.html"; - private static final String FILE_METADATA_JSON = "ro-crate-metadata.json"; - - protected static final String SPECIFICATION_PREFIX = "https://w3id.org/ro/crate/"; - - protected static final String PROP_ABOUT = "about"; - protected static final String PROP_CONTEXT = "@context"; - protected static final String PROP_CONFORMS_TO = "conformsTo"; - protected static final String PROP_GRAPH = "@graph"; - protected static final String PROP_HAS_PART = "hasPart"; - protected static final String PROP_ID = "@id"; - - private final ReaderStrategy reader; - - public RoCrateReader(ReaderStrategy reader) { - this.reader = reader; - } - - /** - * This function will read the location (using one of the specified - * strategies) and then build the relation between the entities. - * - * @param source the input stream to read from - * - * @return the read RO-crate - */ - public RoCrate readCrate(InputStream source) { - RoCrate result = null; - if (reader instanceof StreamReaderStrategy streamReaderStrategy) { - ObjectNode metadata = streamReaderStrategy.readMetadataJson(source); - File content = streamReaderStrategy.readContent(source); - HashSet usedFiles = new HashSet<>(); - usedFiles.add(content.toPath().resolve(FILE_METADATA_JSON).toFile().getPath()); - usedFiles.add(content.toPath().resolve(FILE_PREVIEW_HTML).toFile().getPath()); - usedFiles.add(content.toPath().resolve(FILE_PREVIEW_FILES).toFile().getPath()); - result = rebuildCrate(metadata, content, usedFiles); - } else { - logger.error("Provided writer does not implement StreamReaderStrategy. Please use 'readCrate(String location)'."); - } - return result; - } - - /** - * This function will read the location (using one of the specified - * strategies) and then build the relation between the entities. - * - * @param location the location of the ro-crate to be read - * @return the read RO-crate - */ - public RoCrate readCrate(String location) { - // get the ro-crate-medata.json - ObjectNode metadataJson = reader.readMetadataJson(location); - // get the content of the crate - File files = reader.readContent(location); - - // this set will contain the files that are associated with entities - HashSet usedFiles = new HashSet<>(); - usedFiles.add(files.toPath().resolve(FILE_METADATA_JSON).toFile().getPath()); - usedFiles.add(files.toPath().resolve(FILE_PREVIEW_HTML).toFile().getPath()); - usedFiles.add(files.toPath().resolve(FILE_PREVIEW_FILES).toFile().getPath()); - return rebuildCrate(metadataJson, files, usedFiles); - } - - private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet usedFiles) { - if (metadataJson == null) { - logger.error("Metadata JSON is null, cannot rebuild crate"); - return null; - } - if (files == null) { - logger.error("Content files directory is null, cannot rebuild crate"); - return null; - } - JsonNode context = metadataJson.get(PROP_CONTEXT); - - CrateMetadataContext crateContext = new RoCrateMetadataContext(context); - RoCrateUnsafe crate = new RoCrateUnsafe(); - crate.setMetadataContext(crateContext); - JsonNode graph = metadataJson.get(PROP_GRAPH); - - if (graph.isArray()) { - moveRootEntitiesFromGraphToCrate(crate, (ArrayNode) graph); - RootDataEntity root = crate.getRootDataEntity(); - if (root != null) { - Set dataEntityIds = getDataEntityIds(root, graph); - for (JsonNode entityJson : graph) { - String eId = unpackId(entityJson); - if (dataEntityIds.contains(eId)) { - // data entity - DataEntity.DataEntityBuilder dataEntity = new DataEntity.DataEntityBuilder() - .setAll(entityJson.deepCopy()); - - // Handle data entities with corresponding file - checkFolderHasFile(entityJson.get(PROP_ID).asText(), files).ifPresent(file -> { - usedFiles.add(file.getPath()); - dataEntity.setLocationWithExceptions(file.toPath()) - .setId(file.getName()); - }); - - crate.addDataEntityWithoutRootHasPart(dataEntity.build()); - } else { - // contextual entity - crate.addContextualEntity( - new ContextualEntity.ContextualEntityBuilder() - .setAll(entityJson.deepCopy()) - .build()); - } - } - } - } - - Collection untrackedFiles = Arrays.stream( - Optional.ofNullable(files.listFiles()).orElse(new File[0])) - .filter(f -> !usedFiles.contains(f.getPath())) - .collect(Collectors.toSet()); - - crate.setUntrackedFiles(untrackedFiles); - Validator defaultValidation = new Validator(new JsonSchemaValidation()); - defaultValidation.validate(crate); - return crate; - } - - /** - * Extracts graph connections from top to bottom. - *

- * Example: (connections.get(parent) -> children) - * - * @param graph the ArrayNode with all Entities. - * @return the graph connections. - */ - protected Map> makeEntityGraph(JsonNode graph) { - Map> connections = new HashMap<>(); - - Map idToNodes = new HashMap<>(); - StreamSupport.stream(graph.spliterator(), false) - .forEach(jsonNode -> idToNodes.put(unpackId(jsonNode), jsonNode)); - - for (JsonNode entityNode : graph) { - String currentId = unpackId(entityNode); - StreamSupport.stream(entityNode.path("hasPart").spliterator(), false) - .map(this::unpackId) - .map(s -> idToNodes.getOrDefault(s, null)) - .filter(Objects::nonNull) - .forEach(child -> connections.computeIfAbsent(currentId, key -> new HashSet<>()) - .add(unpackId(child))); - StreamSupport.stream(entityNode.path("isPartOf").spliterator(), false) - .map(this::unpackId) - .map(s -> idToNodes.getOrDefault(s, null)) - .filter(Objects::nonNull) - .forEach(parent -> connections.computeIfAbsent(unpackId(parent), key -> new HashSet<>()) - .add(currentId)); - } - return connections; - } - - protected Set getDataEntityIds(RootDataEntity root, JsonNode graph) { - if (root == null) { - return Set.of(); - } - Map> network = makeEntityGraph(graph); - Set directDataEntities = new HashSet<>(root.hasPart); - - Stack processingQueue = new Stack<>(); - processingQueue.addAll(directDataEntities); - Set result = new HashSet<>(); - - while (!processingQueue.empty()) { - String currentId = processingQueue.pop(); - result.add(currentId); - network.getOrDefault(currentId, new HashSet<>()).stream() - .filter(subId -> !result.contains(subId)) // avoid loops! - .forEach(subId -> { - result.add(subId); - processingQueue.add(subId); - }); - } - return result; - } - - protected String unpackId(JsonNode node) { - if (node.isTextual()) { - return node.asText(); - } else /*if (node.isObject())*/ { - return node.path(PROP_ID).asText(); - } - } - - protected Optional checkFolderHasFile(String filepathOrId, File folder) { - if (IdentifierUtils.isUrl(filepathOrId)) { - return Optional.empty(); - } - return IdentifierUtils.decode(filepathOrId) - .map(decoded -> folder.toPath().resolve(decoded).toFile()) - .filter(File::exists); - } - - /** - * Moves the descriptor and the root entity from the graph to the crate. - *

- * Extracts the root data entity and the Metadata File Descriptor from the - * graph and inserts them into the crate object. It also deletes it from the - * graph. We will need the root dataset to distinguish between data entities - * and contextual entities. - * - * @param crate the crate, which will receive the entities, if available in - * the graph. - * @param graph the graph of the Metadata JSON file, where the entities are - * extracted and removed from. - */ - protected void moveRootEntitiesFromGraphToCrate(RoCrate crate, ArrayNode graph) { - Optional maybeDescriptor = getMetadataDescriptor(graph); - - maybeDescriptor.ifPresent(descriptor -> { - setCrateDescriptor(crate, descriptor); - JsonUtilFunctions.removeJsonNodeFromArrayNode(graph, descriptor); - - Optional maybeRoot = extractRoot(graph, descriptor); - - maybeRoot.ifPresent(root -> { - Set hasPartIds = extractHasPartIds(root); - - crate.setRootDataEntity( - new RootDataEntity.RootDataEntityBuilder() - .setAll(root.deepCopy()) - .setHasPart(hasPartIds) - .build()); - - JsonUtilFunctions.removeJsonNodeFromArrayNode(graph, root); - }); - }); - } - - /** - * Find the metadata descriptor. - *

- * Currently prefers algorithm of version 1.1 over the one of 1.2-DRAFT. - * - * @param graph the graph to search the descriptor in. - * @return the metadata descriptor of the crate. - */ - protected Optional getMetadataDescriptor(ArrayNode graph) { - boolean isParallel = graph.size() > PARALLELIZATION_THRESHOLD; - // use the algorithm described here: - // https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity - Optional maybeDescriptor = StreamSupport.stream(graph.spliterator(), isParallel) - // "2. if the conformsTo property is a URI that starts with - // https://w3id.org/ro/crate/" - .filter(node -> node.path(PROP_CONFORMS_TO).path(PROP_ID).asText().startsWith(SPECIFICATION_PREFIX)) - // "3. from this entity’s about object keep the @id URI as variable root" - .filter(node -> node.path(PROP_ABOUT).path(PROP_ID).isTextual()) - // There should be only one descriptor. If multiple exist, we take the first - // one. - .findFirst(); - return maybeDescriptor.or(() - -> // from https://www.researchobject.org/ro-crate/1.2-DRAFT/root-data-entity.html#finding-the-root-data-entity - StreamSupport.stream(graph.spliterator(), isParallel) - .filter(node -> node.path(PROP_ID).asText().equals(FILE_METADATA_JSON)) - .findFirst() - ); - } - - /** - * Extracts the root entity from the graph, using the information from the - * descriptor. - *

- * Basically implements step 5 of the algorithm described here: - * - * https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity - * - * - * @param graph the graph from the metadata JSON-LD file - * @param descriptor the RO-Crate descriptor - * @return the root entity, if found - */ - private Optional extractRoot(ArrayNode graph, JsonNode descriptor) { - String rootId = descriptor.get(PROP_ABOUT).get(PROP_ID).asText(); - boolean isParallel = graph.size() > PARALLELIZATION_THRESHOLD; - return StreamSupport.stream(graph.spliterator(), isParallel) - // root is an object (filter + conversion) - .filter(JsonNode::isObject) - .map(JsonNode::deepCopy) - // "5. if the entity has an @id URI that matches root return it" - .filter(node -> node.path(PROP_ID).asText().equals(rootId)) - .findFirst(); - } - - private Set extractHasPartIds(ObjectNode root) { - JsonNode hasPartNode = root.path(PROP_HAS_PART); - boolean isParallel = hasPartNode.isArray() && hasPartNode.size() > PARALLELIZATION_THRESHOLD; - Set hasPartIds = StreamSupport.stream(hasPartNode.spliterator(), isParallel) - .map(hasPart -> hasPart.path(PROP_ID).asText()) - .filter(text -> !text.isBlank()) - .collect(Collectors.toSet()); - if (hasPartIds.isEmpty() && hasPartNode.path(PROP_ID).isTextual()) { - hasPartIds.add(hasPartNode.path(PROP_ID).asText()); - } - return hasPartIds; - } - - private void setCrateDescriptor(RoCrate crate, JsonNode descriptor) { - ContextualEntity descriptorEntity = new ContextualEntity.ContextualEntityBuilder() - .setAll(descriptor.deepCopy()) - .build(); - crate.setJsonDescriptor(descriptorEntity); +@Deprecated(since = "2.1.0", forRemoval = true) +public class RoCrateReader extends CrateReader { + public RoCrateReader(GenericReaderStrategy reader) { + super(reader); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java deleted file mode 100644 index 3dfcc243..00000000 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/StreamReaderStrategy.java +++ /dev/null @@ -1,64 +0,0 @@ -package edu.kit.datamanager.ro_crate.reader; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import edu.kit.datamanager.ro_crate.writer.StreamWriterStrategy; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import org.slf4j.LoggerFactory; - -/** - * Interface for reading RO-Crate metadata and content from input streams. - * - * @author jejkal - */ -public interface StreamReaderStrategy extends ReaderStrategy { - - org.slf4j.Logger logger = LoggerFactory.getLogger(StreamReaderStrategy.class); - - /** - * Default override of readMetadataJson interface from ReaderStrategy. The - * override assumes, that location is a file, which is used as input stream. - * If this assumption is not true, this call will fail. - * - * @param location The source, which is supposed to be a file. - * - * @return the RO-Crate metadata as ObjectNode - */ - @Override - default ObjectNode readMetadataJson(String location) { - ObjectNode result = null; - try { - result = readMetadataJson(new FileInputStream(new File(location))); - } catch (FileNotFoundException ex) { - logger.error("Failed read crate from source " + location, ex); - } - return result; - } - - /** - * Default override of readContent interface from ReaderStrategy. The - * override assumes, that location is a file, which is used as input stream. - * If this assumption is not true, this call will fail. - * - * @param location The source, which is supposed to be a file. - * - * @return the RO-Crate content as file, i.e., a folder - */ - @Override - default File readContent(String location) { - File result = null; - try { - result = readContent(new FileInputStream(new File(location))); - } catch (FileNotFoundException ex) { - logger.error("Failed read crate from source " + location, ex); - } - return result; - } - - ObjectNode readMetadataJson(InputStream source); - - File readContent(InputStream source); - -} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java index da60798b..ad9bbb01 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipReader.java @@ -1,18 +1,7 @@ package edu.kit.datamanager.ro_crate.reader; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; - -import java.io.File; -import java.io.IOException; -import java.util.UUID; import java.nio.file.Path; -import net.lingala.zip4j.ZipFile; -import org.apache.commons.io.FileUtils; - /** * A ReaderStrategy implementation which reads from ZipFiles. *

@@ -28,17 +17,18 @@ * folder after extraction. Use RoCrateWriter to export it so some * persistent location and possibly read it from there, if required. Or use * the ZipWriter to write it back to its source. + * + * @deprecated Use {@link ZipStrategy} instead. */ -public class ZipReader implements ReaderStrategy { - - protected final String ID = UUID.randomUUID().toString(); - protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipReader/%s/", ID)); - protected boolean isExtracted = false; +@Deprecated(since = "2.1.0", forRemoval = true) +public class ZipReader extends ZipStrategy { /** * Crates a ZipReader with the default configuration as described in the class documentation. */ - public ZipReader() {} + public ZipReader() { + super(); + } /** * Creates a ZipReader which will extract the contents temporary @@ -52,77 +42,6 @@ public ZipReader() {} * will have UUIDs as their names. */ public ZipReader(Path folderPath, boolean shallAddUuidSubfolder) { - if (shallAddUuidSubfolder) { - this.temporaryFolder = folderPath.resolve(ID); - } else { - this.temporaryFolder = folderPath; - } - } - - /** - * @return the identifier which may be used as the name for a subfolder in the temporary directory. - */ - public String getID() { - return ID; - } - - /** - * @return the folder (considered temporary) where the zipped crate will be or has been extracted to. - */ - public Path getTemporaryFolder() { - return temporaryFolder; - } - - /** - * @return whether the crate has already been extracted into the temporary folder. - */ - public boolean isExtracted() { - return isExtracted; - } - - private void readCrate(String location) { - try { - File folder = temporaryFolder.toFile(); - // ensure the directory is clean - if (folder.isDirectory()) { - FileUtils.cleanDirectory(folder); - } else if (folder.isFile()) { - FileUtils.delete(folder); - } - // extract - try (ZipFile zf = new ZipFile(location)) { - zf.extractAll(temporaryFolder.toAbsolutePath().toString()); - this.isExtracted = true; - } - // register deletion on exit - FileUtils.forceDeleteOnExit(folder); - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Override - public ObjectNode readMetadataJson(String location) { - if (!isExtracted) { - this.readCrate(location); - } - - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - File jsonMetadata = temporaryFolder.resolve("ro-crate-metadata.json").toFile(); - - try { - return objectMapper.readTree(jsonMetadata).deepCopy(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - @Override - public File readContent(String location) { - if (!isExtracted) { - this.readCrate(location); - } - return temporaryFolder.toFile(); + super(folderPath, shallAddUuidSubfolder); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java new file mode 100644 index 00000000..0d6381a6 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStrategy.java @@ -0,0 +1,126 @@ +package edu.kit.datamanager.ro_crate.reader; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import net.lingala.zip4j.ZipFile; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.UUID; + +/** + * A ReaderStrategy implementation which reads from ZipFiles. + *

+ * May be used as a dependency for CrateReader. It will unzip + * the ZipFile in a path relative to the directory this application runs in. + * By default, it will be `./.tmp/ro-crate-java/zipReader/$UUID/`. + *

+ * NOTE: The resulting crate may refer to these temporary files. Therefore, + * these files are only being deleted before the JVM exits. If you need to free + * space because your application is long-running or creates a lot of + * crates, you may use the getters to retrieve information which will help + * you to clean up manually. Keep in mind that crates may refer to this + * folder after extraction. Use RoCrateWriter to export it so some + * persistent location and possibly read it from there, if required. Or use + * the ZipWriter to write it back to its source. + */ +public class ZipStrategy implements GenericReaderStrategy { + + protected final String ID = UUID.randomUUID().toString(); + protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipReader/%s/", ID)); + protected boolean isExtracted = false; + + /** + * Crates a ZipReader with the default configuration as described in the class documentation. + */ + public ZipStrategy() {} + + /** + * Creates a ZipReader which will extract the contents temporary + * to the given location instead of the default location. + * + * @param folderPath the custom directory to extract + * content to for temporary access. + * @param shallAddUuidSubfolder if true, the reader will extract + * into subdirectories of the given + * directory. These subdirectories + * will have UUIDs as their names. + */ + public ZipStrategy(Path folderPath, boolean shallAddUuidSubfolder) { + if (shallAddUuidSubfolder) { + this.temporaryFolder = folderPath.resolve(ID); + } else { + this.temporaryFolder = folderPath; + } + } + + /** + * @return the identifier which may be used as the name for a subfolder in the temporary directory. + */ + public String getID() { + return ID; + } + + /** + * @return the folder (considered temporary) where the zipped crate will be or has been extracted to. + */ + public Path getTemporaryFolder() { + return temporaryFolder; + } + + /** + * @return whether the crate has already been extracted into the temporary folder. + */ + public boolean isExtracted() { + return isExtracted; + } + + private void readCrate(String location) { + try { + File folder = temporaryFolder.toFile(); + // ensure the directory is clean + if (folder.isDirectory()) { + FileUtils.cleanDirectory(folder); + } else if (folder.isFile()) { + FileUtils.delete(folder); + } + // extract + try (ZipFile zf = new ZipFile(location)) { + zf.extractAll(temporaryFolder.toAbsolutePath().toString()); + this.isExtracted = true; + } + // register deletion on exit + FileUtils.forceDeleteOnExit(folder); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public ObjectNode readMetadataJson(String location) { + if (!isExtracted) { + this.readCrate(location); + } + + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + File jsonMetadata = temporaryFolder.resolve("ro-crate-metadata.json").toFile(); + + try { + return objectMapper.readTree(jsonMetadata).deepCopy(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public File readContent(String location) { + if (!isExtracted) { + this.readCrate(location); + } + return temporaryFolder.toFile(); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java similarity index 93% rename from src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java rename to src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java index b71212bb..cb4f53af 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategy.java @@ -13,6 +13,8 @@ import net.lingala.zip4j.io.inputstream.ZipInputStream; import net.lingala.zip4j.model.LocalFileHeader; import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A ZIP file reader implementation of the StreamReaderStrategy interface. @@ -21,8 +23,9 @@ * * @author jejkal */ -public class ZipStreamReader implements StreamReaderStrategy { +public class ZipStreamStrategy implements GenericReaderStrategy { + private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); protected final String ID = UUID.randomUUID().toString(); protected Path temporaryFolder = Path.of(String.format("./.tmp/ro-crate-java/zipStreamReader/%s/", ID)); protected boolean isExtracted = false; @@ -31,7 +34,7 @@ public class ZipStreamReader implements StreamReaderStrategy { * Crates a ZipStreamReader with the default configuration as described in * the class documentation. */ - public ZipStreamReader() { + public ZipStreamStrategy() { } /** @@ -44,7 +47,7 @@ public ZipStreamReader() { * subdirectories of the given directory. These subdirectories will have * UUIDs as their names. */ - public ZipStreamReader(Path folderPath, boolean shallAddUuidSubfolder) { + public ZipStreamStrategy(Path folderPath, boolean shallAddUuidSubfolder) { if (shallAddUuidSubfolder) { this.temporaryFolder = folderPath.resolve(ID); } else { diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java similarity index 92% rename from src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java index 05ca63d5..2e19172a 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java @@ -15,12 +15,13 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.Charset; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -class ZipStreamReaderTest { +class ZipStreamStrategyTest { @Test void testReadingBasicCrate(@TempDir Path temp) throws IOException { @@ -36,7 +37,7 @@ void testReadingBasicCrate(@TempDir Path temp) throws IOException { File zipFile = zipPath.toFile(); assertTrue(zipFile.isFile()); - RoCrateReader roCrateReader = new RoCrateReader(new ZipStreamReader()); + CrateReader roCrateReader = Readers.newZipStreamReader(); Crate res = roCrateReader.readCrate(new FileInputStream(zipFile)); HelpFunctions.compareTwoCrateJson(roCrate, res); } @@ -66,7 +67,7 @@ void testWithFile(@TempDir Path temp) throws IOException { // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, zipPath.toFile().getAbsolutePath()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipStreamReader()); + CrateReader roCrateFolderReader = Readers.newZipStreamReader(); Crate res = roCrateFolderReader.readCrate(new FileInputStream(zipPath.toFile())); HelpFunctions.compareTwoCrateJson(roCrate, res); @@ -95,7 +96,7 @@ void TestWithFileWithLocation(@TempDir Path temp) throws IOException { // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, zipPath.toString()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipStreamReader()); + CrateReader roCrateFolderReader = Readers.newZipStreamReader(); Crate res = roCrateFolderReader.readCrate(new FileInputStream(zipPath.toFile())); Path locationSource = temp.resolve("expected"); @@ -133,7 +134,7 @@ void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, zipPath.toString()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipStreamReader()); + CrateReader roCrateFolderReader = Readers.newZipStreamReader(); Crate res = roCrateFolderReader.readCrate(new FileInputStream(zipPath.toFile())); Path locationSource = temp.resolve("expected"); @@ -175,24 +176,24 @@ void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException assertTrue(zipFile.isFile()); Path differentFolder = temp.resolve("differentFolder"); - ZipStreamReader readerType = new ZipStreamReader(differentFolder, true); + ZipStreamStrategy readerType = new ZipStreamStrategy(differentFolder, true); assertFalse(readerType.isExtracted()); assertEquals(readerType.getTemporaryFolder().getFileName().toString(), readerType.getID()); assertTrue(readerType.getTemporaryFolder().startsWith(differentFolder)); - RoCrateReader roCrateFolderReader = new RoCrateReader(readerType); + CrateReader roCrateFolderReader = new CrateReader<>(readerType); Crate crate = roCrateFolderReader.readCrate(new FileInputStream(zipFile)); assertTrue(readerType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate); { // try it again without the UUID subfolder and test if the directory is being cleaned up (using coverage). - ZipStreamReader newReaderType = new ZipStreamReader(differentFolder, false); + ZipStreamStrategy newReaderType = new ZipStreamStrategy(differentFolder, false); assertFalse(newReaderType.isExtracted()); assertNotEquals(newReaderType.getTemporaryFolder().getFileName().toString(), newReaderType.getID()); assertTrue(newReaderType.getTemporaryFolder().startsWith(differentFolder)); - RoCrateReader newRoCrateFolderReader = new RoCrateReader(newReaderType); + CrateReader newRoCrateFolderReader = new CrateReader<>(newReaderType); Crate crate2 = newRoCrateFolderReader.readCrate(new FileInputStream(zipFile)); assertTrue(newReaderType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate2); From cd8aa1c0e48af2615feb8f1ef23b5c374e697c47 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 22 Apr 2025 16:05:43 +0200 Subject: [PATCH 52/88] refactor: replace deprecated readers in tests with new ones --- .../ro_crate/crate/BuilderSpec12Test.java | 5 ++-- .../ro_crate/crate/ReadAndWriteTest.java | 9 +++---- .../ro_crate/crate/TestRemoveAddContext.java | 5 ++-- .../ro_crate/crate/realexamples/RealTest.java | 6 ++--- .../crate/realexamples/WorkflowHubTest.java | 6 ++--- .../ro_crate/reader/FolderReaderTest.java | 26 +++++++++---------- .../reader/RoCrateReaderSpec12Test.java | 2 +- .../ro_crate/reader/ZipReaderTest.java | 24 ++++++++--------- .../writer/RoCrateWriterSpec12Test.java | 7 +++-- 9 files changed, 43 insertions(+), 47 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java index ee0d72e0..ccbdd5d3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java @@ -8,6 +8,7 @@ import java.net.URISyntaxException; import java.util.Collection; +import edu.kit.datamanager.ro_crate.reader.Readers; import org.junit.jupiter.api.Test; import com.fasterxml.jackson.databind.JsonNode; @@ -15,8 +16,6 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; -import edu.kit.datamanager.ro_crate.reader.FolderReader; -import edu.kit.datamanager.ro_crate.reader.RoCrateReader; import edu.kit.datamanager.ro_crate.special.CrateVersion; import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; @@ -40,7 +39,7 @@ void testAppendConformsTo() throws URISyntaxException { @Test void testModificationOfDraftCrate() throws URISyntaxException { String path = this.getClass().getResource("/crates/spec-1.2-DRAFT/minimal-with-conformsTo-Array").getPath(); - RoCrate crate = new RoCrateReader(new FolderReader()).readCrate(path); + RoCrate crate = Readers.newFolderReader().readCrate(path); Collection existingProfiles = crate.getProfiles(); profile1 = new URI("https://example.com/myprofile/1.0"); profile2 = new URI("https://example.com/myprofile/2.0"); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java index 2f2e308c..4a39cca3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java @@ -3,10 +3,9 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.preview.CustomPreview; import edu.kit.datamanager.ro_crate.preview.StaticPreview; -import edu.kit.datamanager.ro_crate.reader.FolderReader; -import edu.kit.datamanager.ro_crate.reader.RoCrateReader; +import edu.kit.datamanager.ro_crate.reader.CrateReader; +import edu.kit.datamanager.ro_crate.reader.Readers; import edu.kit.datamanager.ro_crate.writer.FolderWriter; import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; @@ -40,7 +39,7 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); writer.save(crate, writeDir.toAbsolutePath().toString()); - RoCrateReader reader = new RoCrateReader(new FolderReader()); + CrateReader reader = Readers.newFolderReader(); Crate newCrate = reader.readCrate(writeDir.toAbsolutePath().toString()); // the preview files as well as the metadata file should not be included here @@ -51,7 +50,7 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { @Test void testReadCrateWithHasPartHierarchy() { - RoCrateReader reader = new RoCrateReader(new FolderReader()); + CrateReader reader = Readers.newFolderReader(); RoCrate crate = reader.readCrate(ReadAndWriteTest.class.getResource("/crates/hasPartHierarchy").getPath()); assertEquals(1, crate.getAllContextualEntities().size()); assertEquals(6, crate.getAllDataEntities().size()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java index 8f008510..b1f6e21b 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/TestRemoveAddContext.java @@ -1,7 +1,6 @@ package edu.kit.datamanager.ro_crate.crate; -import edu.kit.datamanager.ro_crate.reader.FolderReader; -import edu.kit.datamanager.ro_crate.reader.RoCrateReader; +import edu.kit.datamanager.ro_crate.reader.Readers; import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; @@ -21,7 +20,7 @@ public class TestRemoveAddContext { void setup() { String crateManifestPath = "/crates/extendedContextExample/"; crateManifestPath = Objects.requireNonNull(TestRemoveAddContext.class.getResource(crateManifestPath)).getPath(); - this.crateWithComplexContext = new RoCrateReader(new FolderReader()).readCrate(crateManifestPath); + this.crateWithComplexContext = Readers.newFolderReader().readCrate(crateManifestPath); } @Test diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java index bdc132b8..ae206b7c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/RealTest.java @@ -12,8 +12,8 @@ import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.externalproviders.personprovider.OrcidProvider; -import edu.kit.datamanager.ro_crate.reader.FolderReader; -import edu.kit.datamanager.ro_crate.reader.RoCrateReader; +import edu.kit.datamanager.ro_crate.reader.CrateReader; +import edu.kit.datamanager.ro_crate.reader.Readers; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; @@ -30,7 +30,7 @@ class RealTest { @Test void testWithIDRCProject(@TempDir Path temp) throws IOException { - RoCrateReader reader = new RoCrateReader(new FolderReader()); + CrateReader reader = Readers.newFolderReader(); final String locationMetadataFile = "/crates/other/idrc_project/ro-crate-metadata.json"; Crate crate = reader.readCrate(RealTest.class.getResource("/crates/other/idrc_project").getPath()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java index 245a0118..1d45bd09 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java @@ -2,8 +2,8 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.reader.RoCrateReader; -import edu.kit.datamanager.ro_crate.reader.ZipReader; +import edu.kit.datamanager.ro_crate.reader.CrateReader; +import edu.kit.datamanager.ro_crate.reader.Readers; import edu.kit.datamanager.ro_crate.writer.FolderWriter; import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; @@ -18,7 +18,7 @@ public class WorkflowHubTest { @Test void testImportZip(@TempDir Path temp) throws IOException { - RoCrateReader reader = new RoCrateReader(new ZipReader()); + CrateReader reader = Readers.newZipPathReader(); Crate crate = reader.readCrate(WorkflowHubTest.class.getResource("/crates/workflowhub/workflow-109-5.crate.zip").getPath()); HelpFunctions.compareCrateJsonToFileInResources(crate, "/crates/workflowhub/workflow1/ro-crate-metadata.json"); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index d4eb91fb..ad40de3b 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -40,8 +40,8 @@ void testReadingBasicCrate(@TempDir Path temp) throws IOException { .build(); Path f = writeMetadataToFile(temp, roCrate); // Read from written file - RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); - RoCrate res = roCrateFolderReader.readCrate(temp.toFile().toString()); + CrateReader reader = Readers.newFolderReader(); + RoCrate res = reader.readCrate(temp.toFile().toString()); // Write metadata again Path r = temp.resolve("output.txt"); FileUtils.touch(r.toFile()); @@ -71,9 +71,9 @@ void testMultipleReads(@TempDir Path temp1, @TempDir Path temp2) throws IOExcept assertEquals(0, c1.getAllContextualEntities().size()); assertEquals(1, c2.getAllContextualEntities().size()); // read both with the same reader - RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); - RoCrate c1_read = roCrateFolderReader.readCrate(temp1.toFile().toString()); - RoCrate c2_read = roCrateFolderReader.readCrate(temp2.toFile().toString()); + CrateReader reader = Readers.newFolderReader(); + RoCrate c1_read = reader.readCrate(temp1.toFile().toString()); + RoCrate c2_read = reader.readCrate(temp2.toFile().toString()); // check that the reference is not the same assertNotEquals(c1, c1_read); assertNotEquals(c2, c2_read); @@ -105,8 +105,8 @@ void testWithFile(@TempDir Path temp) throws IOException { writeMetadataToFile(temp, roCrate); - RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); - RoCrate res = roCrateFolderReader.readCrate(temp.toFile().toString()); + CrateReader reader = Readers.newFolderReader(); + RoCrate res = reader.readCrate(temp.toFile().toString()); HelpFunctions.compareTwoCrateJson(roCrate, res); } @@ -136,8 +136,8 @@ void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { writeMetadataToFile(temp, roCrate); - RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); - Crate res = roCrateFolderReader.readCrate(temp.toFile().toString()); + CrateReader reader = Readers.newFolderReader(); + Crate res = reader.readCrate(temp.toFile().toString()); HelpFunctions.compareTwoCrateJson(roCrate, res); // Make sure we did not print any errors @@ -169,9 +169,9 @@ void TestWithFileWithLocation(@TempDir Path temp) throws IOException { writeMetadataToFile(temp, roCrate); - RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); + CrateReader reader = Readers.newFolderReader(); - Crate res = roCrateFolderReader.readCrate(locationSource.toFile().toString()); + Crate res = reader.readCrate(locationSource.toFile().toString()); Path destinationDir = temp.resolve("result"); FileUtils.forceMkdir(destinationDir.toFile()); @@ -207,12 +207,12 @@ void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { writeMetadataToFile(temp, roCrate); - RoCrateReader roCrateFolderReader = new RoCrateReader(new FolderReader()); + CrateReader reader = Readers.newFolderReader(); Path newFile = temp.resolve("new_file"); FileUtils.writeStringToFile(newFile.toFile(), "fkladjsl;fjasd;lfjda;lkf", Charset.defaultCharset()); - Crate res = roCrateFolderReader.readCrate(locationSource.toFile().toString()); + Crate res = reader.readCrate(locationSource.toFile().toString()); res.addDataEntity(new FileEntity.FileEntityBuilder() .setEncodingFormat("setnew") .setLocationWithExceptions(newFile) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java index 24bdcb54..7b1df7df 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/RoCrateReaderSpec12Test.java @@ -29,7 +29,7 @@ public class RoCrateReaderSpec12Test { @Test void testReadingCrateWithConformsToArray() { String path = this.getClass().getResource("/crates/spec-1.2-DRAFT/minimal-with-conformsTo-Array").getPath(); - Crate crate = new RoCrateReader(new FolderReader()).readCrate(path); + Crate crate = Readers.newFolderReader().readCrate(path); JsonNode conformsTo = crate.getJsonDescriptor().getProperty("conformsTo"); assertTrue(conformsTo.isArray()); assertEquals(2, conformsTo.size()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 33006ff2..0a55166c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -35,8 +35,8 @@ void testReadingBasicCrate(@TempDir Path temp) throws IOException { File zipFile = zipPath.toFile(); assertTrue(zipFile.isFile()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipReader()); - Crate res = roCrateFolderReader.readCrate(zipFile.getAbsolutePath()); + CrateReader reader = Readers.newZipPathReader(); + Crate res = reader.readCrate(zipFile.getAbsolutePath()); HelpFunctions.compareTwoCrateJson(roCrate, res); } @@ -65,8 +65,8 @@ void testWithFile(@TempDir Path temp) throws IOException { // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, zipPath.toFile().getAbsolutePath()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipReader()); - Crate res = roCrateFolderReader.readCrate(zipPath.toFile().getAbsolutePath()); + CrateReader reader = Readers.newZipPathReader(); + Crate res = reader.readCrate(zipPath.toFile().getAbsolutePath()); HelpFunctions.compareTwoCrateJson(roCrate, res); } @@ -94,8 +94,8 @@ void TestWithFileWithLocation(@TempDir Path temp) throws IOException { // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, zipPath.toString()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipReader()); - Crate res = roCrateFolderReader.readCrate(zipPath.toString()); + CrateReader reader = Readers.newZipPathReader(); + Crate res = reader.readCrate(zipPath.toString()); Path locationSource = temp.resolve("expected"); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); @@ -132,8 +132,8 @@ void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { // save the content of the roCrate to the dest zip roCrateZipWriter.save(roCrate, zipPath.toString()); - RoCrateReader roCrateFolderReader = new RoCrateReader(new ZipReader()); - Crate res = roCrateFolderReader.readCrate(zipPath.toFile().getAbsolutePath()); + CrateReader reader = Readers.newZipPathReader(); + Crate res = reader.readCrate(zipPath.toFile().getAbsolutePath()); Path locationSource = temp.resolve("expected"); RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); @@ -174,24 +174,24 @@ void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException assertTrue(zipFile.isFile()); Path differentFolder = temp.resolve("differentFolder"); - ZipReader readerType = new ZipReader(differentFolder, true); + ZipStrategy readerType = new ZipStrategy(differentFolder, true); assertFalse(readerType.isExtracted()); assertEquals(readerType.getTemporaryFolder().getFileName().toString(), readerType.getID()); assertTrue(readerType.getTemporaryFolder().startsWith(differentFolder)); - RoCrateReader roCrateFolderReader = new RoCrateReader(readerType); + CrateReader roCrateFolderReader = new CrateReader<>(readerType); Crate crate = roCrateFolderReader.readCrate(zipFile.getAbsolutePath()); assertTrue(readerType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate); { // try it again without the UUID subfolder and test if the directory is being cleaned up (using coverage). - ZipReader newReaderType = new ZipReader(differentFolder, false); + ZipStrategy newReaderType = new ZipStrategy(differentFolder, false); assertFalse(newReaderType.isExtracted()); assertNotEquals(newReaderType.getTemporaryFolder().getFileName().toString(), newReaderType.getID()); assertTrue(newReaderType.getTemporaryFolder().startsWith(differentFolder)); - RoCrateReader newRoCrateFolderReader = new RoCrateReader(newReaderType); + CrateReader newRoCrateFolderReader = new CrateReader<>(newReaderType); Crate crate2 = newRoCrateFolderReader.readCrate(zipFile.getAbsolutePath()); assertTrue(newReaderType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate2); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java index b6d4ed40..fc4dd016 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java @@ -10,13 +10,12 @@ import java.nio.file.Path; import java.nio.file.Paths; +import edu.kit.datamanager.ro_crate.reader.Readers; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.reader.FolderReader; -import edu.kit.datamanager.ro_crate.reader.RoCrateReader; class RoCrateWriterSpec12Test { @@ -27,7 +26,7 @@ void writeDoesNotModifyTest(@TempDir Path tempDir) throws IOException, URISyntax URL internalOriginalCrateURL = this.getClass().getResource("/" + internalOriginalCratePath); assertNotNull(internalOriginalCrateURL); - Crate crate = new RoCrateReader(new FolderReader()).readCrate(internalOriginalCrateURL.getPath()); + Crate crate = Readers.newFolderReader().readCrate(internalOriginalCrateURL.getPath()); Path targetDir = tempDir.resolve("spec12writeUnmodified"); { @@ -51,7 +50,7 @@ void writeDoesNotModifyTest(@TempDir Path tempDir) throws IOException, URISyntax // original metadata file new File(srcDir.resolve("ro-crate-metadata.json").toString())); // Compare loaded crate object with crate object made of export - Crate crate2 = new RoCrateReader(new FolderReader()).readCrate(targetDir.toString()); + Crate crate2 = Readers.newFolderReader().readCrate(targetDir.toString()); HelpFunctions.compareTwoCrateJson(crate, crate2); } } From 2d669d5e36b778b89fbc5fa295c8163ece8445a8 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 22 Apr 2025 16:06:28 +0200 Subject: [PATCH 53/88] fix(test): add missing assertions about profile presence --- .../edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java index ccbdd5d3..eddd93ee 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/BuilderSpec12Test.java @@ -68,8 +68,8 @@ void testModificationOfDraftCrate() throws URISyntaxException { Collection newProfileState = modifiedCrate.getProfiles(); assertEquals(existingProfiles.size() + 2, newProfileState.size()); // new profiles are present - newProfileState.contains(profile1.toString()); - newProfileState.contains(profile2.toString()); + assertTrue(newProfileState.contains(profile1.toString())); + assertTrue(newProfileState.contains(profile2.toString())); // old profiles are present assertEquals( 0, From 06d2ce63c998afe683b8eb72a2e2a77bba90bc28 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 22 Apr 2025 16:31:44 +0200 Subject: [PATCH 54/88] test: add tests for convenience APIs --- .../ro_crate/reader/ZipReaderTest.java | 15 +++++++++++---- .../ro_crate/reader/ZipStreamStrategyTest.java | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index 0a55166c..cee73355 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -179,8 +179,8 @@ void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException assertEquals(readerType.getTemporaryFolder().getFileName().toString(), readerType.getID()); assertTrue(readerType.getTemporaryFolder().startsWith(differentFolder)); - CrateReader roCrateFolderReader = new CrateReader<>(readerType); - Crate crate = roCrateFolderReader.readCrate(zipFile.getAbsolutePath()); + CrateReader reader = new CrateReader<>(readerType); + Crate crate = reader.readCrate(zipFile.getAbsolutePath()); assertTrue(readerType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate); @@ -191,10 +191,17 @@ void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException assertNotEquals(newReaderType.getTemporaryFolder().getFileName().toString(), newReaderType.getID()); assertTrue(newReaderType.getTemporaryFolder().startsWith(differentFolder)); - CrateReader newRoCrateFolderReader = new CrateReader<>(newReaderType); - Crate crate2 = newRoCrateFolderReader.readCrate(zipFile.getAbsolutePath()); + CrateReader newReader = new CrateReader<>(newReaderType); + Crate crate2 = newReader.readCrate(zipFile.getAbsolutePath()); assertTrue(newReaderType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate2); } + + { + // show we can also do it with the convenience API + CrateReader newReader = Readers.newZipPathReader(differentFolder, false); + Crate crate2 = newReader.readCrate(zipFile.getAbsolutePath()); + HelpFunctions.compareTwoCrateJson(roCrate, crate2); + } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java index 2e19172a..a7d37f7e 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java @@ -181,8 +181,8 @@ void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException assertEquals(readerType.getTemporaryFolder().getFileName().toString(), readerType.getID()); assertTrue(readerType.getTemporaryFolder().startsWith(differentFolder)); - CrateReader roCrateFolderReader = new CrateReader<>(readerType); - Crate crate = roCrateFolderReader.readCrate(new FileInputStream(zipFile)); + CrateReader reader = new CrateReader<>(readerType); + Crate crate = reader.readCrate(new FileInputStream(zipFile)); assertTrue(readerType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate); @@ -193,10 +193,17 @@ void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException assertNotEquals(newReaderType.getTemporaryFolder().getFileName().toString(), newReaderType.getID()); assertTrue(newReaderType.getTemporaryFolder().startsWith(differentFolder)); - CrateReader newRoCrateFolderReader = new CrateReader<>(newReaderType); - Crate crate2 = newRoCrateFolderReader.readCrate(new FileInputStream(zipFile)); + CrateReader reader2 = new CrateReader<>(newReaderType); + Crate crate2 = reader2.readCrate(new FileInputStream(zipFile)); assertTrue(newReaderType.isExtracted()); HelpFunctions.compareTwoCrateJson(roCrate, crate2); } + + { + // show we can also do it with the convenience API + CrateReader reader2 = Readers.newZipStreamReader(differentFolder, false); + Crate crate2 = reader2.readCrate(new FileInputStream(zipFile)); + HelpFunctions.compareTwoCrateJson(roCrate, crate2); + } } } From d44defaf9e2a9e041ee6f078af9da8193c22e098 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 22 Apr 2025 17:13:54 +0200 Subject: [PATCH 55/88] feat: introduce generic readers instead of the StreamReaderStrategy --- .../ro_crate/writer/CrateWriter.java | 30 ++++++++++++ .../writer/GenericWriterStrategy.java | 19 ++++++++ .../ro_crate/writer/RoCrateWriter.java | 48 +++---------------- .../ro_crate/writer/StreamWriterStrategy.java | 36 -------------- .../ro_crate/writer/WriterStrategy.java | 9 ++-- .../datamanager/ro_crate/writer/Writers.java | 19 ++++++++ .../ro_crate/writer/ZipStreamWriter.java | 4 +- .../ro_crate/writer/ZipStreamWriterTest.java | 33 ++++++------- 8 files changed, 97 insertions(+), 101 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java delete mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java new file mode 100644 index 00000000..440be0c4 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/CrateWriter.java @@ -0,0 +1,30 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; +import edu.kit.datamanager.ro_crate.validation.Validator; + +/** + * The class used for writing (exporting) crates. The class uses a strategy + * pattern for writing crates as different formats. (zip, folders, etc.) + */ +public class CrateWriter { + + private final GenericWriterStrategy strategy; + + public CrateWriter(GenericWriterStrategy strategy) { + this.strategy = strategy; + } + + /** + * This method saves the crate to a destination provided. + * + * @param crate the crate to write. + * @param destination the location where the crate should be written. + */ + public void save(Crate crate, DESTINATION destination) { + Validator defaultValidation = new Validator(new JsonSchemaValidation()); + defaultValidation.validate(crate); + this.strategy.save(crate, destination); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java new file mode 100644 index 00000000..6306b576 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/GenericWriterStrategy.java @@ -0,0 +1,19 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; + +/** + * Generic interface for the strategy of the writer class. + * This allows for flexible output types when implementing different writing strategies. + * + * @param the type of the destination parameter + */ +public interface GenericWriterStrategy { + /** + * Saves the given crate to the specified destination. + * + * @param crate The crate to save + * @param destination The destination where the crate should be saved + */ + void save(Crate crate, DESTINATION destination); +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java index 112b4d3a..9337d400 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriter.java @@ -1,51 +1,15 @@ package edu.kit.datamanager.ro_crate.writer; -import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; -import edu.kit.datamanager.ro_crate.validation.Validator; -import java.io.OutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * The class used for writing (exporting) crates. The class uses a strategy * pattern for writing crates as different formats. (zip, folders, etc.) + * + * @deprecated Use {@link CrateWriter} instead. */ -public class RoCrateWriter { - - private static Logger logger = LoggerFactory.getLogger(RoCrateWriter.class); - - private final WriterStrategy writer; - - public RoCrateWriter(WriterStrategy writer) { - this.writer = writer; - } - - /** - * This method saves the crate to a destination provided. - * - * @param crate the crate to write. - * @param destination the location where the crate should be written. - */ - public void save(Crate crate, String destination) { - Validator defaultValidation = new Validator(new JsonSchemaValidation()); - defaultValidation.validate(crate); - this.writer.save(crate, destination); - } +@Deprecated(since = "2.1.0", forRemoval = true) +public class RoCrateWriter extends CrateWriter { - /** - * This method saves the crate to a destination provided. - * - * @param crate the crate to write. - * @param destination the location where the crate should be written. - */ - public void save(Crate crate, OutputStream destination) { - Validator defaultValidation = new Validator(new JsonSchemaValidation()); - defaultValidation.validate(crate); - if (writer instanceof StreamWriterStrategy streamWriterStrategy) { - streamWriterStrategy.save(crate, destination); - } else { - logger.error("Provided writer does not implement StreamWriterStrategy. Please use 'save(Crate crate, String destination)'."); - } + public RoCrateWriter(GenericWriterStrategy writer) { + super(writer); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java deleted file mode 100644 index 04deb659..00000000 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/StreamWriterStrategy.java +++ /dev/null @@ -1,36 +0,0 @@ -package edu.kit.datamanager.ro_crate.writer; - -import edu.kit.datamanager.ro_crate.Crate; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.OutputStream; -import org.slf4j.LoggerFactory; - -/** - * Strategy for writing of crates to streams. - * - * @author jejkal - */ -public interface StreamWriterStrategy extends WriterStrategy { - - static org.slf4j.Logger logger = LoggerFactory.getLogger(StreamWriterStrategy.class); - - /** - * Default override of save interface from WriterStrategy. The override - * assumes, that destination is a file, which is used as output stream. If - * this assumption is not true, this call will fail. - * - * @param crate The crate to write. - * @param destination The destination, which is supposed to be a file. - */ - default void save(Crate crate, String destination) { - try { - save(crate, new FileOutputStream(new File(destination))); - } catch (FileNotFoundException ex) { - logger.error("Failed save crate to destination " + destination, ex); - } - } - - void save(Crate crate, OutputStream destination); -} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriterStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriterStrategy.java index 06be6585..12459673 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriterStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriterStrategy.java @@ -1,13 +1,12 @@ package edu.kit.datamanager.ro_crate.writer; -import edu.kit.datamanager.ro_crate.Crate; - /** * Strategy for writing of crates. * * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 + * + * @deprecated Use {@link GenericWriterStrategy} instead. */ -public interface WriterStrategy { - void save(Crate crate, String destination); -} +@Deprecated(since = "2.1.0", forRemoval = true) +public interface WriterStrategy extends GenericWriterStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java new file mode 100644 index 00000000..0cd90078 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java @@ -0,0 +1,19 @@ +package edu.kit.datamanager.ro_crate.writer; + +import java.io.OutputStream; + +/** + * Utility class for creating instances of different crate writers. + * This class is not meant to be instantiated. + */ +public class Writers { + + /** + * Prevents instantiation of this utility class. + */ + private Writers() {} + + public static CrateWriter newZipStreamWriter() { + return new CrateWriter<>(new ZipStreamWriter()); + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java index 3145c3a1..0d1ee27e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java @@ -21,9 +21,9 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStreamWriter implements StreamWriterStrategy { +public class ZipStreamWriter implements GenericWriterStrategy { - private static Logger logger = LoggerFactory.getLogger(ZipStreamWriter.class); + private static final Logger logger = LoggerFactory.getLogger(ZipStreamWriter.class); @Override public void save(Crate crate, OutputStream destination) { diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java index 2aa39bf3..d17bd785 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java @@ -3,8 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.charset.Charset; import java.nio.file.Path; @@ -78,16 +77,16 @@ void testWritingToZipStream(@TempDir Path tempDir) throws IOException { .setPreview(new AutomaticPreview()) .build(); - // safe the crate in the test.zip file - Path test = tempDir.resolve("test.zip"); // create a Writer for writing RoCrates to zip - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipStreamWriter()); - - - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, test.toFile().getAbsolutePath()); + CrateWriter writer = Writers.newZipStreamWriter(); + // write into destination path + Path destination_path = tempDir.resolve("test.zip"); + OutputStream destination = new FileOutputStream(destination_path.toFile()); + writer.save(roCrate, destination); + + // extract and compare Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(test.toFile())) { + try (ZipFile zf = new ZipFile(destination_path.toFile())) { zf.extractAll(res.toFile().getAbsolutePath()); } assertTrue(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); @@ -154,14 +153,16 @@ void testWritingToZipFail(@TempDir Path tempDir) throws IOException { .setPreview(new AutomaticPreview()) .build(); - // safe the crate in the test.zip file - Path test = tempDir.resolve("test.zip"); // create a Writer for writing RoCrates to zip - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipStreamWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, test.toFile().getAbsolutePath()); + CrateWriter writer = Writers.newZipStreamWriter(); + // write into destination path + Path destination_path = tempDir.resolve("test.zip"); + OutputStream destination = new FileOutputStream(destination_path.toFile()); + writer.save(roCrate, destination); + + // extract and compare Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(test.toFile())) { + try (ZipFile zf = new ZipFile(destination_path.toFile())) { zf.extractAll(res.toFile().getAbsolutePath()); } assertFalse(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); From 2ba9b0152abb3e95b55dc220797e74c92a0c49ad Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 23 Apr 2025 13:42:29 +0200 Subject: [PATCH 56/88] feat: rename strategies from the scheme $(format)Writer to $(format)Strategy and keep old names as deprecated aliases --- .../ro_crate/writer/FolderStrategy.java | 63 +++++++++++++++++ .../ro_crate/writer/FolderWriter.java | 61 ++--------------- .../datamanager/ro_crate/writer/Writers.java | 2 +- .../ro_crate/writer/ZipStrategy.java | 68 +++++++++++++++++++ ...reamWriter.java => ZipStreamStrategy.java} | 4 +- .../ro_crate/writer/ZipWriter.java | 67 ++---------------- ...erTest.java => ZipStreamStrategyTest.java} | 6 +- 7 files changed, 145 insertions(+), 126 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java create mode 100644 src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java rename src/main/java/edu/kit/datamanager/ro_crate/writer/{ZipStreamWriter.java => ZipStreamStrategy.java} (96%) rename src/test/java/edu/kit/datamanager/ro_crate/writer/{ZipStreamWriterTest.java => ZipStreamStrategyTest.java} (97%) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java new file mode 100644 index 00000000..d7638e8b --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -0,0 +1,63 @@ +package edu.kit.datamanager.ro_crate.writer; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * A class for writing a crate to a folder. + * + * @author Nikola Tzotchev on 9.2.2022 г. + * @version 1 + */ +public class FolderStrategy implements GenericWriterStrategy { + + private static Logger logger = LoggerFactory.getLogger(FolderStrategy.class); + + @Override + public void save(Crate crate, String destination) { + File file = new File(destination); + try { + FileUtils.forceMkdir(file); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + + File json = new File(destination, "ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(inputStream, json); + inputStream.close(); + // save also the preview files to the crate destination + if (crate.getPreview() != null) { + crate.getPreview().saveAllToFolder(file); + } + for (var e : crate.getUntrackedFiles()) { + if (e.isDirectory()) { + FileUtils.copyDirectoryToDirectory(e, file); + } else { + FileUtils.copyFileToDirectory(e, file); + } + } + } catch (IOException e) { + logger.error("Error creating destination directory!", e); + } + for (DataEntity dataEntity : crate.getAllDataEntities()) { + try { + dataEntity.savetoFile(file); + } catch (IOException e) { + logger.error("Cannot save " + dataEntity.getId() + " to destination folder!", e); + } + } + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java index ab704181..5730104e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderWriter.java @@ -1,64 +1,11 @@ package edu.kit.datamanager.ro_crate.writer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * A class for writing a crate to a folder. * * @author Nikola Tzotchev on 9.2.2022 г. - * @version 1 + * + * @deprecated Use {@link FolderStrategy} instead. */ -public class FolderWriter implements WriterStrategy { - - private static Logger logger = LoggerFactory.getLogger(FolderWriter.class); - - @Override - public void save(Crate crate, String destination) { - File file = new File(destination); - try { - FileUtils.forceMkdir(file); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); - - File json = new File(destination, "ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(inputStream, json); - inputStream.close(); - // save also the preview files to the crate destination - if (crate.getPreview() != null) { - crate.getPreview().saveAllToFolder(file); - } - for (var e : crate.getUntrackedFiles()) { - if (e.isDirectory()) { - FileUtils.copyDirectoryToDirectory(e, file); - } else { - FileUtils.copyFileToDirectory(e, file); - } - } - } catch (IOException e) { - logger.error("Error creating destination directory!", e); - } - for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.savetoFile(file); - } catch (IOException e) { - logger.error("Cannot save " + dataEntity.getId() + " to destination folder!", e); - } - } - } -} +@Deprecated(since = "2.1.0", forRemoval = true) +public class FolderWriter extends FolderStrategy {} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java index 0cd90078..ea2287bf 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java @@ -14,6 +14,6 @@ public class Writers { private Writers() {} public static CrateWriter newZipStreamWriter() { - return new CrateWriter<>(new ZipStreamWriter()); + return new CrateWriter<>(new ZipStreamStrategy()); } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java new file mode 100644 index 00000000..e677a86e --- /dev/null +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -0,0 +1,68 @@ +package edu.kit.datamanager.ro_crate.writer; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.model.ZipParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * Implementation of the writing strategy to provide a way of writing crates to + * a zip archive. + */ +public class ZipStrategy implements GenericWriterStrategy { + + private static Logger logger = LoggerFactory.getLogger(ZipStrategy.class); + + @Override + public void save(Crate crate, String destination) { + try (ZipFile zipFile = new ZipFile(destination)) { + saveMetadataJson(crate, zipFile); + saveDataEntities(crate, zipFile); + } catch (IOException e) { + // can not close ZipFile (threw Exception) + logger.error("Failed to write ro-crate to destination " + destination + ".", e); + } + } + + private void saveDataEntities(Crate crate, ZipFile zipFile) { + for (DataEntity dataEntity : crate.getAllDataEntities()) { + try { + dataEntity.saveToZip(zipFile); + } catch (ZipException e) { + logger.error("Could not save " + dataEntity.getId() + " to zip file!", e); + } + } + } + + private void saveMetadataJson(Crate crate, ZipFile zipFile) { + try { + // write the metadata.json file + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); + // we create an JsonNode only to have the file written pretty + JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); + String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + // write the ro-crate-metadata + zipFile.addStream(inputStream, zipParameters); + inputStream.close(); + if (crate.getPreview() != null) { + crate.getPreview().saveAllToZip(zipFile); + } + } catch (IOException e) { + logger.error("Exception writing ro-crate-metadata.json file to zip.", e); + } + } +} diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java similarity index 96% rename from src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java rename to src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index 0d1ee27e..146df9f5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -21,9 +21,9 @@ * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. */ -public class ZipStreamWriter implements GenericWriterStrategy { +public class ZipStreamStrategy implements GenericWriterStrategy { - private static final Logger logger = LoggerFactory.getLogger(ZipStreamWriter.class); + private static final Logger logger = LoggerFactory.getLogger(ZipStreamStrategy.class); @Override public void save(Crate crate, OutputStream destination) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java index dff25fd0..f5261003 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipWriter.java @@ -1,69 +1,10 @@ package edu.kit.datamanager.ro_crate.writer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.entities.data.DataEntity; -import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * Implementation of the writing strategy to provide a way of writing crates to * a zip archive. + * + * @deprecated Use {@link ZipStrategy} instead. */ -public class ZipWriter implements WriterStrategy { - - private static Logger logger = LoggerFactory.getLogger(ZipWriter.class); - - @Override - public void save(Crate crate, String destination) { - try (ZipFile zipFile = new ZipFile(destination)) { - saveMetadataJson(crate, zipFile); - saveDataEntities(crate, zipFile); - } catch (IOException e) { - // can not close ZipFile (threw Exception) - logger.error("Failed to write ro-crate to destination " + destination + ".", e); - } - } - - private void saveDataEntities(Crate crate, ZipFile zipFile) { - for (DataEntity dataEntity : crate.getAllDataEntities()) { - try { - dataEntity.saveToZip(zipFile); - } catch (ZipException e) { - logger.error("Could not save " + dataEntity.getId() + " to zip file!", e); - } - } - } - - private void saveMetadataJson(Crate crate, ZipFile zipFile) { - try { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); - // we create an JsonNode only to have the file written pretty - JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); - String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - inputStream.close(); - if (crate.getPreview() != null) { - crate.getPreview().saveAllToZip(zipFile); - } - } catch (IOException e) { - logger.error("Exception writing ro-crate-metadata.json file to zip.", e); - } - } -} +@Deprecated(since = "2.1.0", forRemoval = true) +public class ZipWriter extends ZipStrategy {} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java similarity index 97% rename from src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index d17bd785..54b03452 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -21,7 +21,7 @@ /** * @author jejkal */ -class ZipStreamWriterTest { +class ZipStreamStrategyTest { @Test void testWritingToZipStream(@TempDir Path tempDir) throws IOException { @@ -31,7 +31,7 @@ void testWritingToZipStream(@TempDir Path tempDir) throws IOException { // the .json of our crate InputStream fileJson= - ZipStreamWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + ZipStreamStrategyTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); // fill the expected directory with files and dirs @@ -104,7 +104,7 @@ void testWritingToZipFail(@TempDir Path tempDir) throws IOException { // the .json of our crate InputStream fileJson= - ZipStreamWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + ZipStreamStrategyTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); // fill the expected directory with files and dirs From 4ee8c9cc6019efc7538e36f7f7e6713ebb0c2a91 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 23 Apr 2025 13:46:59 +0200 Subject: [PATCH 57/88] refactor: apply smaller linter suggestions about best practices --- .../edu/kit/datamanager/ro_crate/writer/FolderStrategy.java | 2 +- .../java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java | 2 +- .../edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java | 2 +- .../kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index d7638e8b..a4d1b63d 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -23,7 +23,7 @@ */ public class FolderStrategy implements GenericWriterStrategy { - private static Logger logger = LoggerFactory.getLogger(FolderStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(FolderStrategy.class); @Override public void save(Crate crate, String destination) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index e677a86e..c944aad3 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -22,7 +22,7 @@ */ public class ZipStrategy implements GenericWriterStrategy { - private static Logger logger = LoggerFactory.getLogger(ZipStrategy.class); + private static final Logger logger = LoggerFactory.getLogger(ZipStrategy.class); @Override public void save(Crate crate, String destination) { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java index 146df9f5..818c064c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategy.java @@ -41,7 +41,7 @@ private void saveDataEntities(Crate crate, ZipOutputStream zipStream) { try { dataEntity.saveToStream(zipStream); } catch (IOException e) { - logger.error("Could not save " + dataEntity.getId() + " to zip stream!", e); + logger.error("Could not save {} to zip stream!", dataEntity.getId(), e); } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index 54b03452..d8ea8c6c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -15,6 +15,7 @@ import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; import net.lingala.zip4j.ZipFile; import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -32,6 +33,7 @@ void testWritingToZipStream(@TempDir Path tempDir) throws IOException { // the .json of our crate InputStream fileJson= ZipStreamStrategyTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + Assertions.assertNotNull(fileJson); // fill the expected directory with files and dirs @@ -105,6 +107,7 @@ void testWritingToZipFail(@TempDir Path tempDir) throws IOException { // the .json of our crate InputStream fileJson= ZipStreamStrategyTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); + Assertions.assertNotNull(fileJson); // fill the expected directory with files and dirs From b341acf222efaf15b1a185b32f0679ef2798fcd6 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Wed, 23 Apr 2025 13:49:52 +0200 Subject: [PATCH 58/88] feat: add new writer factory methods for folder, zip stream, and zip file --- .../datamanager/ro_crate/writer/Writers.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java index ea2287bf..3d469698 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java @@ -13,7 +13,30 @@ public class Writers { */ private Writers() {} + /** + * Creates a new instance of a crate writer that writes to a folder. + * + * @return a new instance of {@link CrateWriter} for writing to a folder + */ + public static CrateWriter newFolderWriter() { + return new CrateWriter<>(new FolderStrategy()); + } + + /** + * Creates a new instance of a crate writer that writes to a zip stream. + * + * @return a new instance of {@link CrateWriter} for writing to a zip stream + */ public static CrateWriter newZipStreamWriter() { return new CrateWriter<>(new ZipStreamStrategy()); } + + /** + * Creates a new instance of a crate writer that writes to a zip file. + * + * @return a new instance of {@link CrateWriter} for writing to a zip file + */ + public static CrateWriter newZipFileWriter() { + return new CrateWriter<>(new ZipStrategy()); + } } From 37ca46d16a46f196613ea61a876624c76f213989 Mon Sep 17 00:00:00 2001 From: Thomas Jejkal Date: Wed, 23 Apr 2025 16:35:52 +0200 Subject: [PATCH 59/88] Fix #101, updated ror provider to v2, adapted tests --- .../edu/kit/datamanager/ro_crate/RoCrate.java | 16 +++- .../ro_crate/entities/data/DataEntity.java | 2 - .../validation/JsonSchemaValidation.java | 2 +- .../organizationprovider/RorProvider.java | 91 ++++++++++++------- .../entity_field_structure_schema.json | 1 - .../externalproviders/RorProviderTest.java | 41 +++++---- 6 files changed, 93 insertions(+), 60 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java index 2e6db32b..476fde7f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java @@ -10,9 +10,13 @@ import edu.kit.datamanager.ro_crate.entities.AbstractEntity; import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity; import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor; +import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity; import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity.DataEntityBuilder; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity; import edu.kit.datamanager.ro_crate.externalproviders.dataentities.ImportFromDataCite; +import edu.kit.datamanager.ro_crate.externalproviders.organizationprovider.RorProvider; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import edu.kit.datamanager.ro_crate.payload.CratePayload; import edu.kit.datamanager.ro_crate.payload.RoCratePayload; @@ -22,9 +26,12 @@ import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions; import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation; import edu.kit.datamanager.ro_crate.validation.Validator; +import edu.kit.datamanager.ro_crate.writer.FolderWriter; +import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; import java.io.File; import java.net.URI; +import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -58,22 +65,27 @@ public void setRoCratePreview(CratePreview preview) { this.roCratePreview = preview; } + @Override public void setMetadataContext(CrateMetadataContext metadataContext) { this.metadataContext = metadataContext; } + @Override public ContextualEntity getJsonDescriptor() { return jsonDescriptor; } + @Override public void setJsonDescriptor(ContextualEntity jsonDescriptor) { this.jsonDescriptor = jsonDescriptor; } - + + @Override public RootDataEntity getRootDataEntity() { return rootDataEntity; } - + + @Override public void setRootDataEntity(RootDataEntity rootDataEntity) { this.rootDataEntity = rootDataEntity; } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 50ca8d2a..93ca2ae8 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -8,9 +8,7 @@ import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Path; import java.util.ArrayList; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java index beaca0fa..067f5e2e 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/validation/JsonSchemaValidation.java @@ -68,7 +68,7 @@ public boolean validateEntity(JsonNode entity) { @Override public boolean validateFieldOfEntity(JsonNode field) { Set errors = this.entityFieldSchema.validate(field); - if (errors.size() != 0) { + if (!errors.isEmpty()) { ObjectMapper objectMapper = MyObjectMapper.getMapper(); System.err.println("The property: "); try { diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java index c3158515..3885c4b1 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java @@ -1,11 +1,14 @@ package edu.kit.datamanager.ro_crate.externalproviders.organizationprovider; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; import java.io.IOException; +import java.util.ArrayList; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -20,45 +23,63 @@ */ public class RorProvider { - private static Logger logger = LoggerFactory.getLogger(RorProvider.class); + private static Logger logger = LoggerFactory.getLogger(RorProvider.class); - private RorProvider() {} + private RorProvider() { + } + + /** + * The method that parses a ror entry to a crate entity. + * + * @param url the url of the ror entry. + * @return the created Organization entity. + */ + public static OrganizationEntity getOrganization(String url) { + if (!url.startsWith("https://ror.org/")) { + throw new IllegalArgumentException("Should provide ror url"); + } + String newUrl = "https://api.ror.org/v2/organizations/" + url.replaceAll("https://ror.org/", ""); + HttpGet request = new HttpGet(newUrl); - /** - * The method that parses a ror entry to a crate entity. - * - * @param url the url of the ror entry. - * @return the created Organization entity. - */ - public static OrganizationEntity getOrganization(String url) { - if (!url.startsWith("https://ror.org/")) { - throw new IllegalArgumentException("Should provide ror url"); + try ( + CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpResponse response = httpClient.execute(request);) { + boolean isError = response.getStatusLine().getStatusCode() != HttpStatus.SC_OK; + if (isError) { + String errorMessage = String.format("Identifier not found: %s", response.getStatusLine().toString()); + logger.error(errorMessage); + return null; + } + ObjectNode jsonNode = MyObjectMapper.getMapper().readValue(response.getEntity().getContent(), + ObjectNode.class); + + return new OrganizationEntity.OrganizationEntityBuilder() + .setId(jsonNode.get("id").asText()) + .addProperty("name", getOrganizationNameV2(jsonNode.get("names"))) + .addProperty("email", jsonNode.get("email_address")) + .addProperty("url", jsonNode.get("id").asText()) + .build(); + } catch (IOException e) { + String errorMessage = String.format("IO error: %s", e.getMessage()); + logger.error(errorMessage); + } + return null; } - String newUrl = "https://api.ror.org/organizations/" + url.replaceAll("https://ror.org/", ""); - HttpGet request = new HttpGet(newUrl); - try ( - CloseableHttpClient httpClient = HttpClients.createDefault(); - CloseableHttpResponse response = httpClient.execute(request); - ) { - boolean isError = response.getStatusLine().getStatusCode() != HttpStatus.SC_OK; - if (isError) { - String errorMessage = String.format("Identifier not found: %s", response.getStatusLine().toString()); - logger.error(errorMessage); - return null; + private static String getOrganizationNameV2(JsonNode node) { + if (node.isArray()) { + for (JsonNode n : node) { + if (n.has("types") && n.get("types").isArray()) { + ArrayList l = new ObjectMapper().convertValue(n.get("types"), ArrayList.class); + if (l.contains("ror_display")) { + return n.get("value").asText(); + } + } + } + //fallback + return node.get(0).get("value").asText(); + } else { + return node.asText(); } - ObjectNode jsonNode = MyObjectMapper.getMapper().readValue(response.getEntity().getContent(), - ObjectNode.class); - return new OrganizationEntity.OrganizationEntityBuilder() - .setId(jsonNode.get("id").asText()) - .addProperty("name", jsonNode.get("name")) - .addProperty("email", jsonNode.get("email_address")) - .addProperty("url", jsonNode.get("links")) - .build(); - } catch (IOException e) { - String errorMessage = String.format("IO error: %s", e.getMessage()); - logger.error(errorMessage); } - return null; - } + } diff --git a/src/main/resources/json_schemas/entity_field_structure_schema.json b/src/main/resources/json_schemas/entity_field_structure_schema.json index 864422e9..d2112ebb 100644 --- a/src/main/resources/json_schemas/entity_field_structure_schema.json +++ b/src/main/resources/json_schemas/entity_field_structure_schema.json @@ -12,7 +12,6 @@ {"type": "boolean"}, {"type": "null"} ] - } }, { diff --git a/src/test/java/edu/kit/datamanager/ro_crate/externalproviders/RorProviderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/externalproviders/RorProviderTest.java index 5880ed72..9300e157 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/externalproviders/RorProviderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/externalproviders/RorProviderTest.java @@ -5,6 +5,7 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity; import edu.kit.datamanager.ro_crate.externalproviders.organizationprovider.RorProvider; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -18,23 +19,25 @@ */ public class RorProviderTest { - @Test - void testExternalRorProvider() throws IOException { - OrganizationEntity organizationEntity = RorProvider.getOrganization("https://ror.org/04t3en479"); - assertNotNull(organizationEntity); - HelpFunctions.compareEntityWithFile(organizationEntity, "/json/entities/contextual/rorkit.json"); - } - - @Test - void testInvalidRorUrl() { - assertThrows(IllegalArgumentException.class, () -> { - RorProvider.getOrganization("https://notror.org/04t3en479"); - }); - } - - @Test - void testInvalidRorId() { - OrganizationEntity organizationEntity = RorProvider.getOrganization("https://ror.org/42"); - assertNull(organizationEntity); - } + @Test + void testExternalRorProvider() throws IOException { + OrganizationEntity organizationEntity = RorProvider.getOrganization("https://ror.org/04t3en479"); + assertNotNull(organizationEntity); + Assertions.assertEquals("https://ror.org/04t3en479", organizationEntity.getProperty("@id").asText()); + Assertions.assertEquals("https://ror.org/04t3en479", organizationEntity.getProperty("url").asText()); + Assertions.assertEquals("Karlsruhe Institute of Technology", organizationEntity.getProperty("name").asText()); + } + + @Test + void testInvalidRorUrl() { + assertThrows(IllegalArgumentException.class, () -> { + RorProvider.getOrganization("https://notror.org/04t3en479"); + }); + } + + @Test + void testInvalidRorId() { + OrganizationEntity organizationEntity = RorProvider.getOrganization("https://ror.org/42"); + assertNull(organizationEntity); + } } From 4d572f056b34e57fc4036daed5649d78d0ec029d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Thu, 24 Apr 2025 14:48:44 +0200 Subject: [PATCH 60/88] feat: rename newZipFileWriter to newZipPathWriter for consistency with newZipPathReader --- src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java index 3d469698..5b691ece 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/Writers.java @@ -36,7 +36,7 @@ public static CrateWriter newZipStreamWriter() { * * @return a new instance of {@link CrateWriter} for writing to a zip file */ - public static CrateWriter newZipFileWriter() { + public static CrateWriter newZipPathWriter() { return new CrateWriter<>(new ZipStrategy()); } } From bef5f67fb5351e591d6f7f6444c21f3bf2f0ca2c Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Fri, 25 Apr 2025 16:01:17 +0200 Subject: [PATCH 61/88] refactor(test): Unify reader tests and use new API --- .../ro_crate/reader/CrateReaderTest.java | 230 ++++++++++++++++++ .../ro_crate/reader/FolderReaderTest.java | 227 +++-------------- .../ro_crate/reader/ZipReaderTest.java | 215 ++-------------- .../reader/ZipStreamStrategyTest.java | 219 ++--------------- 4 files changed, 323 insertions(+), 568 deletions(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java new file mode 100644 index 00000000..bcf8948d --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/CrateReaderTest.java @@ -0,0 +1,230 @@ +package edu.kit.datamanager.ro_crate.reader; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.*; +import java.nio.charset.Charset; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Abstract class for testing crate readers. + * + * @param the source type of the reader strategy. Even though each implementation knows this T, + * we can't use it everywhere we'd like to as the code here needs to be generic. + * We therefore rely on methods to take a path (as we always assume local testing). + * Streams, for example, will therefore need to stream from/to a file. + * This parameter is only required to satisfy the generic reader strategy. + * @param the type of the reader strategy + */ +abstract class CrateReaderTest> { + + protected static RoCrate.RoCrateBuilder newBaseCrate() { + return new RoCrate.RoCrateBuilder( + "minimal", + "minimal RO_crate", + "2024", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ); + } + + protected static FileEntity newDataEntity(Path filePath) throws IllegalArgumentException { + return new FileEntity.FileEntityBuilder() + .setLocationWithExceptions(filePath) + .setId(filePath.toFile().getName()) + .addProperty("name", "Survey responses") + .addProperty("contentSize", "26452") + .addProperty("encodingFormat", "text/csv") + .build(); + } + + /** + * Saves the crate with the writer fitting to the reader of {@link #readCrate(Path)}. + * + * @param crate the crate to save + * @param target the target path to the save location + * @throws IOException if an error occurs while saving the crate + */ + abstract protected void saveCrate(Crate crate, Path target) throws IOException; + + /** + * Reads the crate with the reader fitting to the writer of {@link #saveCrate(Crate, Path)}. + * @param source the source path to the crate + * @return the read crate + * @throws IOException if an error occurs while reading the crate + */ + abstract protected Crate readCrate(Path source) throws IOException; + + /** + * Creates a new reader strategy with a non-default temporary directory (if supported, default otherwise). + * + * @param tmpDirectory the temporary directory to use + * @param useUuidSubfolder whether to create a UUID subfolder under the temporary directory + * @return a new reader strategy + */ + abstract protected READER_STRATEGY newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder); + + /** + * Reads the crate using the provided reader strategy. + * + * @param strategy the reader strategy to use + * @param source the source path to the crate + * @return the read crate + * @throws IOException if an error occurs while reading the crate + */ + abstract protected Crate readCrate(READER_STRATEGY strategy, Path source) throws IOException; + + @Test + void testReadingBasicCrate(@TempDir Path temp) throws IOException { + + RoCrate roCrate = newBaseCrate().build(); + Path zipPath = temp.resolve("result.zip"); + this.saveCrate(roCrate, zipPath); + Crate importedCrate = this.readCrate(zipPath); + HelpFunctions.compareTwoCrateJson(roCrate, importedCrate); + } + + @Test + void testWithFile(@TempDir Path temp) throws IOException { + Path csvPath = temp.resolve("survey-responses-2019.csv"); + FileUtils.touch(csvPath.toFile()); + FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); + RoCrate rawCrate = newBaseCrate() + .addDataEntity(newDataEntity(csvPath)) + .build(); + + assertEquals(1, rawCrate.getAllDataEntities().size()); + + Path zipPath = temp.resolve("result.zip"); + this.saveCrate(rawCrate, zipPath); + Crate importedCrate = this.readCrate(zipPath); + + HelpFunctions.compareTwoCrateJson(rawCrate, importedCrate); + } + + @Test + void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { + // This URL will be encoded because of whitespaces + Path csvPath = temp.resolve("survey responses 2019.csv"); + FileUtils.touch(csvPath.toFile()); + FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); + RoCrate rawCrate = newBaseCrate() + .addDataEntity(newDataEntity(csvPath)) + .build(); + + DataEntity rawEntity = rawCrate.getAllDataEntities().iterator().next(); + assertTrue(rawEntity.getId().contains("survey")); + assertFalse(rawEntity.getId().contains(" ")); + assertEquals(1, rawCrate.getAllDataEntities().size()); + + Path zipPath = temp.resolve("result.zip"); + this.saveCrate(rawCrate, zipPath); + Crate importedCrate = this.readCrate(zipPath); + + DataEntity importedEntity = importedCrate.getAllDataEntities().iterator().next(); + assertTrue(importedEntity.getId().contains("survey")); + assertFalse(importedEntity.getId().contains(" ")); + assertEquals(1, importedCrate.getAllDataEntities().size()); + + HelpFunctions.compareTwoCrateJson(rawCrate, importedCrate); + } + + @Test + void TestWithFileWithLocation(@TempDir Path temp) throws IOException { + Path csvPath = temp.resolve("survey-responses-2019.csv"); + FileUtils.writeStringToFile(csvPath.toFile(), "Dummy content", Charset.defaultCharset()); + RoCrate rawCrate = newBaseCrate() + .addDataEntity(newDataEntity(csvPath)) + .setPreview(null)//disable preview to allow to compare folders before and after + .build(); + + // write to zip file and read via zip stream + Path zipPath = temp.resolve("result.zip"); + this.saveCrate(rawCrate, zipPath); + Crate importedCrate = this.readCrate(zipPath); + + // write raw crate and imported crate to folders and compare the results + Path rawCrateTarget = temp.resolve("rawCrateSaved"); + Path importedCrateTarget = temp.resolve("importedCrateSaved"); + { + // write raw crate and imported crate to two different directories + CrateWriter writer = Writers.newFolderWriter(); + writer.save(rawCrate, rawCrateTarget.toString()); + writer.save(importedCrate, importedCrateTarget.toString()); + } + + assertTrue(HelpFunctions.compareTwoDir(rawCrateTarget.toFile(), importedCrateTarget.toFile())); + HelpFunctions.compareTwoCrateJson(rawCrate, importedCrate); + } + + @Test + void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { + Path csvPath = temp.resolve("file.csv"); + FileUtils.writeStringToFile(csvPath.toFile(), "fakecsv.1", Charset.defaultCharset()); + RoCrate rawCrate = newBaseCrate() + .addDataEntity(newDataEntity(csvPath)) + .build(); + + // write to zip file and import via zip stream + Path zipPath = temp.resolve("result.zip"); + this.saveCrate(rawCrate, zipPath); + Crate importedCrate = this.readCrate(zipPath); + { + // modify the imported crate + Path newFile = temp.resolve("new_file"); + FileUtils.writeStringToFile(newFile.toFile(), "Some file content", Charset.defaultCharset()); + importedCrate.addDataEntity(new FileEntity.FileEntityBuilder() + .setEncodingFormat("setnew") + .setLocationWithExceptions(newFile) + .setId("new_file") + .build()); + } + // write raw crate to a folder + Path rawCrateTarget = temp.resolve("rawCrateSaved"); + Path importedCrateTarget = temp.resolve("importedCrateSaved"); + { + // write raw crate and imported crate to two different directories + CrateWriter writer = Writers.newFolderWriter(); + writer.save(rawCrate, rawCrateTarget.toString()); + writer.save(importedCrate, importedCrateTarget.toFile().toString()); + } + // assert the folders are different + assertFalse(HelpFunctions.compareTwoDir(rawCrateTarget.toFile(), importedCrateTarget.toFile())); + HelpFunctions.compareTwoMetadataJsonNotEqual(rawCrate, importedCrate); + // assert the importedCrateTarget folder contains newFile + assertTrue(importedCrateTarget.resolve("new_file").toFile().isFile()); + } + + @Test + void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { + RoCrate rawCrate = newBaseCrate().build(); + + // Write to zip file + Path zipPath = temp.resolve("result.zip"); + this.saveCrate(rawCrate, zipPath); + + // read again and compare using custom path for temporary extraction folder + // (if available, otherwise uses default) + Path differentFolder = temp.resolve("differentFolder"); + READER_STRATEGY strategy = this.newReaderStrategyWithTmp(differentFolder, true); + Crate importedCrate = this.readCrate(strategy, zipPath); + HelpFunctions.compareTwoCrateJson(rawCrate, importedCrate); + + { + // try it again without the UUID subfolder and test if the directory is being cleaned up (for coverage). + READER_STRATEGY strategyWithoutSubfolder = this.newReaderStrategyWithTmp(differentFolder, false); + Crate crate = this.readCrate(strategyWithoutSubfolder, zipPath); + HelpFunctions.compareTwoCrateJson(rawCrate, crate); + } + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java index ad40de3b..21850edd 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/FolderReaderTest.java @@ -4,19 +4,12 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.contextual.PersonEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; - -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.PrintStream; -import java.nio.charset.Charset; import java.nio.file.Path; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.*; @@ -24,49 +17,54 @@ * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderReaderTest { +class FolderReaderTest extends CrateReaderTest { - private Path writeMetadataToFile(Path temp, RoCrate c1) throws IOException { - // Write metadata to file - Path f = temp.resolve("ro-crate-metadata.json"); - FileUtils.touch(f.toFile()); - FileUtils.writeStringToFile(f.toFile(), c1.getJsonMetadata(), Charset.defaultCharset()); - return f; + @Override + protected void saveCrate(Crate crate, Path target) { + Writers.newFolderWriter().save(crate, target.toAbsolutePath().toString()); + assertTrue(target.toFile().isDirectory()); } - @Test - void testReadingBasicCrate(@TempDir Path temp) throws IOException { - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .build(); - Path f = writeMetadataToFile(temp, roCrate); - // Read from written file - CrateReader reader = Readers.newFolderReader(); - RoCrate res = reader.readCrate(temp.toFile().toString()); - // Write metadata again - Path r = temp.resolve("output.txt"); - FileUtils.touch(r.toFile()); - FileUtils.writeStringToFile(r.toFile(), res.getJsonMetadata(), Charset.defaultCharset()); - // See if it is the same - assertTrue(FileUtils.contentEquals(f.toFile(), r.toFile())); + @Override + protected Crate readCrate(Path source) throws IOException { + return Readers.newFolderReader().readCrate(source.toAbsolutePath().toString()); } + @Override + protected FolderStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + // This strategy does not support a non-default temporary directory + // and will always use the default one. + // It also has no state we could make assertions on. + return new FolderStrategy(); + } + + @Override + protected Crate readCrate(FolderStrategy strategy, Path source) throws IOException { + return new CrateReader<>(strategy) + .readCrate(source.toAbsolutePath().toString()); + } + + /** + * The folder reader is state-less, so we should be able to read multiple crates + * with the same instance. + */ @Test void testMultipleReads(@TempDir Path temp1, @TempDir Path temp2) throws IOException { String id = "https://orcid.org/0000-0001-6121-5409"; PersonEntity person = new PersonEntity.PersonEntityBuilder() - .setId(id) - .setContactPoint("mailto:tim.luckett@uts.edu.au") - .setAffiliation("https://ror.org/03f0f6041") - .setFamilyName("Luckett") - .setGivenName("Tim") - .addProperty("name", "Tim Luckett") - .build(); + .setId(id) + .setContactPoint("mailto:tim.luckett@uts.edu.au") + .setAffiliation("https://ror.org/03f0f6041") + .setFamilyName("Luckett") + .setGivenName("Tim") + .addProperty("name", "Tim Luckett") + .build(); RoCrate c1 = new RoCrate.RoCrateBuilder("mini", "test", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/").build(); RoCrate c2 = new RoCrate.RoCrateBuilder("other", "with file", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addContextualEntity(person) - .build(); - writeMetadataToFile(temp1, c1); - writeMetadataToFile(temp2, c2); + .addContextualEntity(person) + .build(); + this.saveCrate(c1, temp1); + this.saveCrate(c2, temp2); // some first checks... assertEquals(0, c1.getAllContextualEntities().size()); assertEquals(1, c2.getAllContextualEntities().size()); @@ -82,149 +80,4 @@ void testMultipleReads(@TempDir Path temp1, @TempDir Path temp2) throws IOExcept assertEquals(1, c2_read.getAllContextualEntities().size()); HelpFunctions.compareTwoMetadataJsonNotEqual(c1_read, c2_read); } - - @Test - void testWithFile(@TempDir Path temp) throws IOException { - Path cvs = temp.resolve("survey-responses-2019.csv"); - FileUtils.touch(cvs.toFile()); - FileUtils.writeStringToFile(cvs.toFile(), "fkdjaflkjfla", Charset.defaultCharset()); - - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .setLocationWithExceptions(cvs) - .setId(cvs.toFile().getName()) - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .build() - ) - .build(); - - assertEquals(1, roCrate.getAllDataEntities().size()); - - writeMetadataToFile(temp, roCrate); - - CrateReader reader = Readers.newFolderReader(); - RoCrate res = reader.readCrate(temp.toFile().toString()); - HelpFunctions.compareTwoCrateJson(roCrate, res); - } - - @Test - void testWithFileUrlEncoded(@TempDir Path temp) throws IOException { - - // get the std output redirected, so we can see if there is something written - PrintStream standardOut = System.out; - ByteArrayOutputStream outputStreamCaptor = new ByteArrayOutputStream(); - System.setOut(new PrintStream(outputStreamCaptor)); - - Path csv = temp.resolve("survey responses 2019.csv"); // This URL will be encoded because of whitespaces - FileUtils.touch(csv.toFile()); - FileUtils.writeStringToFile(csv.toFile(), "fkdjaflkjfla", Charset.defaultCharset()); - - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .setLocationWithExceptions(csv) - .setId(csv.toFile().getName()) - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .build() - ) - .build(); - - writeMetadataToFile(temp, roCrate); - - CrateReader reader = Readers.newFolderReader(); - Crate res = reader.readCrate(temp.toFile().toString()); - HelpFunctions.compareTwoCrateJson(roCrate, res); - - // Make sure we did not print any errors - assertEquals("", outputStreamCaptor.toString().trim()); - System.setOut(standardOut); - } - - @Test - void TestWithFileWithLocation(@TempDir Path temp) throws IOException { - Path file = temp.resolve("survey-responses-2019.csv"); - FileUtils.writeStringToFile(file.toFile(), "fakecsv.1", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .setLocationWithExceptions(file) - .setId("survey-responses-2019.csv") - .build() - ) - .setPreview(null)//disable preview to allow to compare folders before and after - .build(); - Path locationSource = temp.resolve("src"); - FileUtils.forceMkdir(locationSource.toFile()); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - - writer.save(roCrate, locationSource.toFile().toString()); - - writeMetadataToFile(temp, roCrate); - - CrateReader reader = Readers.newFolderReader(); - - Crate res = reader.readCrate(locationSource.toFile().toString()); - - Path destinationDir = temp.resolve("result"); - FileUtils.forceMkdir(destinationDir.toFile()); - - writer.save(res, destinationDir.toFile().toString()); - - // that copies the directory locally to see its content - //FileUtils.copyDirectory(locationSource.toFile(), new File("test")); - assertTrue(HelpFunctions.compareTwoDir(locationSource.toFile(), destinationDir.toFile())); - HelpFunctions.compareTwoCrateJson(roCrate, res); - } - - @Test - void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { - Path file = temp.resolve("file.csv"); - FileUtils.writeStringToFile(file.toFile(), "fakecsv.1", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .setLocationWithExceptions(file) - .setId("survey-responses-2019.csv") - .build() - ) - .build(); - Path locationSource = temp.resolve("src"); - FileUtils.forceMkdir(locationSource.toFile()); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - - writer.save(roCrate, locationSource.toFile().toString()); - - writeMetadataToFile(temp, roCrate); - - CrateReader reader = Readers.newFolderReader(); - - Path newFile = temp.resolve("new_file"); - FileUtils.writeStringToFile(newFile.toFile(), "fkladjsl;fjasd;lfjda;lkf", Charset.defaultCharset()); - - Crate res = reader.readCrate(locationSource.toFile().toString()); - res.addDataEntity(new FileEntity.FileEntityBuilder() - .setEncodingFormat("setnew") - .setLocationWithExceptions(newFile) - .setId("new_file") - .build()); - - Path destinationDir = temp.resolve("result"); - FileUtils.forceMkdir(destinationDir.toFile()); - - writer.save(res, destinationDir.toFile().toString()); - - assertFalse(HelpFunctions.compareTwoDir(locationSource.toFile(), destinationDir.toFile())); - HelpFunctions.compareTwoMetadataJsonNotEqual(roCrate, res); - } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java index cee73355..3611c834 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipReaderTest.java @@ -1,207 +1,44 @@ package edu.kit.datamanager.ro_crate.reader; import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; -import edu.kit.datamanager.ro_crate.writer.ZipWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -class ZipReaderTest { - - @Test - void testReadingBasicCrate(@TempDir Path temp) throws IOException { - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toString()); - - File zipFile = zipPath.toFile(); - assertTrue(zipFile.isFile()); +class ZipReaderTest extends CrateReaderTest { - CrateReader reader = Readers.newZipPathReader(); - Crate res = reader.readCrate(zipFile.getAbsolutePath()); - HelpFunctions.compareTwoCrateJson(roCrate, res); + @Override + protected void saveCrate(Crate crate, Path target) { + Writers.newZipPathWriter().save(crate, target.toAbsolutePath().toString()); + assertTrue(target.toFile().isFile()); } - @Test - void testWithFile(@TempDir Path temp) throws IOException { - Path cvs = temp.resolve("survey-responses-2019.csv"); - FileUtils.touch(cvs.toFile()); - FileUtils.writeStringToFile(cvs.toFile(), "fkdjaflkjfla", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .setLocationWithExceptions(cvs) - .setId(cvs.toFile().getName()) - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .build() - ) - .build(); - - assertEquals(1, roCrate.getAllDataEntities().size()); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toFile().getAbsolutePath()); - - CrateReader reader = Readers.newZipPathReader(); - Crate res = reader.readCrate(zipPath.toFile().getAbsolutePath()); - - HelpFunctions.compareTwoCrateJson(roCrate, res); + @Override + protected Crate readCrate(Path source) throws IOException { + return Readers.newZipPathReader().readCrate(source.toAbsolutePath().toString()); } - @Test - void TestWithFileWithLocation(@TempDir Path temp) throws IOException { - Path file = temp.resolve("survey-responses-2019.csv"); - FileUtils.writeStringToFile(file.toFile(), "fakecsv.1", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .setLocationWithExceptions(file) - .setId("survey-responses-2019.csv") - .build() - ) - .setPreview(null)//disable preview to allow to compare folders before and after - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toString()); - - CrateReader reader = Readers.newZipPathReader(); - Crate res = reader.readCrate(zipPath.toString()); - - Path locationSource = temp.resolve("expected"); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(roCrate, locationSource.toString()); - - Path destinationDir = temp.resolve("result"); - FileUtils.forceMkdir(destinationDir.toFile()); - writer.save(res, destinationDir.toString()); - - // that copies the directory locally to see its content - assertTrue(HelpFunctions.compareTwoDir(locationSource.toFile(), destinationDir.toFile())); - HelpFunctions.compareTwoCrateJson(roCrate, res); - } - - @Test - void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { - Path file = temp.resolve("file.csv"); - FileUtils.writeStringToFile(file.toFile(), "fakecsv.1", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .setLocationWithExceptions(file) - .setId("survey-responses-2019.csv") - .build() - ) - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toString()); - - CrateReader reader = Readers.newZipPathReader(); - Crate res = reader.readCrate(zipPath.toFile().getAbsolutePath()); - - Path locationSource = temp.resolve("expected"); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(roCrate, locationSource.toString()); - - Path newFile = temp.resolve("new_file"); - FileUtils.writeStringToFile(newFile.toFile(), "fkladjsl;fjasd;lfjda;lkf", Charset.defaultCharset()); - - res.addDataEntity(new FileEntity.FileEntityBuilder() - .setEncodingFormat("setnew") - .setLocationWithExceptions(newFile) - .setId("new_file") - .build()); - - Path destinationDir = temp.resolve("result"); - FileUtils.forceMkdir(destinationDir.toFile()); - writer.save(res, destinationDir.toFile().toString()); - - assertFalse(HelpFunctions.compareTwoDir(locationSource.toFile(), destinationDir.toFile())); - HelpFunctions.compareTwoMetadataJsonNotEqual(roCrate, res); - } - - @Test - void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { - RoCrate roCrate = new RoCrate.RoCrateBuilder( - "minimal", - "minimal RO_crate", - "2024", - "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - roCrateZipWriter.save(roCrate, zipPath.toString()); - - File zipFile = zipPath.toFile(); - assertTrue(zipFile.isFile()); - - Path differentFolder = temp.resolve("differentFolder"); - ZipStrategy readerType = new ZipStrategy(differentFolder, true); - assertFalse(readerType.isExtracted()); - assertEquals(readerType.getTemporaryFolder().getFileName().toString(), readerType.getID()); - assertTrue(readerType.getTemporaryFolder().startsWith(differentFolder)); - - CrateReader reader = new CrateReader<>(readerType); - Crate crate = reader.readCrate(zipFile.getAbsolutePath()); - assertTrue(readerType.isExtracted()); - HelpFunctions.compareTwoCrateJson(roCrate, crate); - - { - // try it again without the UUID subfolder and test if the directory is being cleaned up (using coverage). - ZipStrategy newReaderType = new ZipStrategy(differentFolder, false); - assertFalse(newReaderType.isExtracted()); - assertNotEquals(newReaderType.getTemporaryFolder().getFileName().toString(), newReaderType.getID()); - assertTrue(newReaderType.getTemporaryFolder().startsWith(differentFolder)); - - CrateReader newReader = new CrateReader<>(newReaderType); - Crate crate2 = newReader.readCrate(zipFile.getAbsolutePath()); - assertTrue(newReaderType.isExtracted()); - HelpFunctions.compareTwoCrateJson(roCrate, crate2); + @Override + protected ZipStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + ZipStrategy strategy = new ZipStrategy(tmpDirectory, useUuidSubfolder); + assertFalse(strategy.isExtracted()); + if (useUuidSubfolder) { + assertEquals(strategy.getTemporaryFolder().getFileName().toString(), strategy.getID()); + } else { + assertEquals(strategy.getTemporaryFolder().getFileName().toString(), tmpDirectory.getFileName().toString()); } + assertTrue(strategy.getTemporaryFolder().startsWith(tmpDirectory)); + return strategy; + } - { - // show we can also do it with the convenience API - CrateReader newReader = Readers.newZipPathReader(differentFolder, false); - Crate crate2 = newReader.readCrate(zipFile.getAbsolutePath()); - HelpFunctions.compareTwoCrateJson(roCrate, crate2); - } + @Override + protected Crate readCrate(ZipStrategy strategy, Path source) throws IOException { + Crate importedCrate = new CrateReader<>(strategy) + .readCrate(source.toAbsolutePath().toString()); + assertTrue(strategy.isExtracted()); + return importedCrate; } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java index a7d37f7e..a2548408 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java @@ -1,209 +1,44 @@ package edu.kit.datamanager.ro_crate.reader; import edu.kit.datamanager.ro_crate.Crate; -import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; -import edu.kit.datamanager.ro_crate.writer.ZipWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; +import java.io.*; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -class ZipStreamStrategyTest { - - @Test - void testReadingBasicCrate(@TempDir Path temp) throws IOException { - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toString()); - - File zipFile = zipPath.toFile(); - assertTrue(zipFile.isFile()); - CrateReader roCrateReader = Readers.newZipStreamReader(); - Crate res = roCrateReader.readCrate(new FileInputStream(zipFile)); - HelpFunctions.compareTwoCrateJson(roCrate, res); +class ZipStreamStrategyTest extends CrateReaderTest { + @Override + protected void saveCrate(Crate crate, Path target) throws IOException { + Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); + assertTrue(target.toFile().isFile()); } - @Test - void testWithFile(@TempDir Path temp) throws IOException { - Path cvs = temp.resolve("survey-responses-2019.csv"); - FileUtils.touch(cvs.toFile()); - FileUtils.writeStringToFile(cvs.toFile(), "fkdjaflkjfla", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .setLocationWithExceptions(cvs) - .setId(cvs.toFile().getName()) - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .build() - ) - .build(); - - assertEquals(1, roCrate.getAllDataEntities().size()); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toFile().getAbsolutePath()); - - CrateReader roCrateFolderReader = Readers.newZipStreamReader(); - Crate res = roCrateFolderReader.readCrate(new FileInputStream(zipPath.toFile())); - - HelpFunctions.compareTwoCrateJson(roCrate, res); + @Override + protected Crate readCrate(Path source) throws IOException { + return Readers.newZipStreamReader().readCrate(new FileInputStream(source.toFile())); } - @Test - void TestWithFileWithLocation(@TempDir Path temp) throws IOException { - Path file = temp.resolve("survey-responses-2019.csv"); - FileUtils.writeStringToFile(file.toFile(), "fakecsv.1", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .setLocationWithExceptions(file) - .setId("survey-responses-2019.csv") - .build() - ) - .setPreview(null)//disable preview to allow to compare folders before and after - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toString()); - - CrateReader roCrateFolderReader = Readers.newZipStreamReader(); - Crate res = roCrateFolderReader.readCrate(new FileInputStream(zipPath.toFile())); - - Path locationSource = temp.resolve("expected"); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(roCrate, locationSource.toString()); - - Path destinationDir = temp.resolve("result"); - FileUtils.forceMkdir(destinationDir.toFile()); - writer.save(res, destinationDir.toString()); - - // that copies the directory locally to see its content - assertTrue(HelpFunctions.compareTwoDir(locationSource.toFile(), destinationDir.toFile())); - HelpFunctions.compareTwoCrateJson(roCrate, res); - } - - @Test - void TestWithFileWithLocationAddEntity(@TempDir Path temp) throws IOException { - Path file = temp.resolve("file.csv"); - FileUtils.writeStringToFile(file.toFile(), "fakecsv.1", Charset.defaultCharset()); - RoCrate roCrate = new RoCrate.RoCrateBuilder("minimal", "minimal RO_crate", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Survey responses") - .addProperty("contentSize", "26452") - .addProperty("encodingFormat", "text/csv") - .setLocationWithExceptions(file) - .setId("survey-responses-2019.csv") - .build() - ) - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, zipPath.toString()); - - CrateReader roCrateFolderReader = Readers.newZipStreamReader(); - Crate res = roCrateFolderReader.readCrate(new FileInputStream(zipPath.toFile())); - - Path locationSource = temp.resolve("expected"); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(roCrate, locationSource.toString()); - - Path newFile = temp.resolve("new_file"); - FileUtils.writeStringToFile(newFile.toFile(), "fkladjsl;fjasd;lfjda;lkf", Charset.defaultCharset()); - - res.addDataEntity(new FileEntity.FileEntityBuilder() - .setEncodingFormat("setnew") - .setLocationWithExceptions(newFile) - .setId("new_file") - .build()); - - Path destinationDir = temp.resolve("result"); - FileUtils.forceMkdir(destinationDir.toFile()); - writer.save(res, destinationDir.toFile().toString()); - - assertFalse(HelpFunctions.compareTwoDir(locationSource.toFile(), destinationDir.toFile())); - HelpFunctions.compareTwoMetadataJsonNotEqual(roCrate, res); - } - - @Test - void testReadingBasicCrateWithCustomPath(@TempDir Path temp) throws IOException { - RoCrate roCrate = new RoCrate.RoCrateBuilder( - "minimal", - "minimal RO_crate", - "2024", - "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .build(); - - Path zipPath = temp.resolve("result.zip"); - - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - roCrateZipWriter.save(roCrate, zipPath.toString()); - - File zipFile = zipPath.toFile(); - assertTrue(zipFile.isFile()); - - Path differentFolder = temp.resolve("differentFolder"); - ZipStreamStrategy readerType = new ZipStreamStrategy(differentFolder, true); - assertFalse(readerType.isExtracted()); - assertEquals(readerType.getTemporaryFolder().getFileName().toString(), readerType.getID()); - assertTrue(readerType.getTemporaryFolder().startsWith(differentFolder)); - - CrateReader reader = new CrateReader<>(readerType); - Crate crate = reader.readCrate(new FileInputStream(zipFile)); - assertTrue(readerType.isExtracted()); - HelpFunctions.compareTwoCrateJson(roCrate, crate); - - { - // try it again without the UUID subfolder and test if the directory is being cleaned up (using coverage). - ZipStreamStrategy newReaderType = new ZipStreamStrategy(differentFolder, false); - assertFalse(newReaderType.isExtracted()); - assertNotEquals(newReaderType.getTemporaryFolder().getFileName().toString(), newReaderType.getID()); - assertTrue(newReaderType.getTemporaryFolder().startsWith(differentFolder)); - - CrateReader reader2 = new CrateReader<>(newReaderType); - Crate crate2 = reader2.readCrate(new FileInputStream(zipFile)); - assertTrue(newReaderType.isExtracted()); - HelpFunctions.compareTwoCrateJson(roCrate, crate2); + @Override + protected ZipStreamStrategy newReaderStrategyWithTmp(Path tmpDirectory, boolean useUuidSubfolder) { + ZipStreamStrategy strategy = new ZipStreamStrategy(tmpDirectory, useUuidSubfolder); + assertFalse(strategy.isExtracted()); + if (useUuidSubfolder) { + assertEquals(strategy.getTemporaryFolder().getFileName().toString(), strategy.getID()); + } else { + assertEquals(strategy.getTemporaryFolder().getFileName().toString(), tmpDirectory.getFileName().toString()); } + assertTrue(strategy.getTemporaryFolder().startsWith(tmpDirectory)); + return strategy; + } - { - // show we can also do it with the convenience API - CrateReader reader2 = Readers.newZipStreamReader(differentFolder, false); - Crate crate2 = reader2.readCrate(new FileInputStream(zipFile)); - HelpFunctions.compareTwoCrateJson(roCrate, crate2); - } + @Override + protected Crate readCrate(ZipStreamStrategy strategy, Path source) throws IOException { + Crate importedCrate = new CrateReader<>(strategy) + .readCrate(new FileInputStream(source.toFile())); + assertTrue(strategy.isExtracted()); + return importedCrate; } } From a4e0ad782e5fcdaa41adb0faeed2e8422d978a31 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 28 Apr 2025 11:57:48 +0200 Subject: [PATCH 62/88] refactor(test): rename ZipStreamStrategyTest to ZipStreamReaderTest for consistency --- .../{ZipStreamStrategyTest.java => ZipStreamReaderTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/edu/kit/datamanager/ro_crate/reader/{ZipStreamStrategyTest.java => ZipStreamReaderTest.java} (94%) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java similarity index 94% rename from src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java rename to src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java index a2548408..33436c6c 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/reader/ZipStreamReaderTest.java @@ -9,7 +9,7 @@ import static org.junit.jupiter.api.Assertions.*; -class ZipStreamStrategyTest extends CrateReaderTest { +class ZipStreamReaderTest extends CrateReaderTest { @Override protected void saveCrate(Crate crate, Path target) throws IOException { Writers.newZipStreamWriter().save(crate, new FileOutputStream(target.toFile())); From 6fd11dae5c5bfe8725a420d0089455c296e0d513 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 28 Apr 2025 19:32:27 +0200 Subject: [PATCH 63/88] refactor(test): unify writer tests --- .../ro_crate/writer/CrateWriterTest.java | 251 ++++++++++++++++++ .../ro_crate/writer/FolderWriterTest.java | 206 +------------- .../writer/ZipStreamStrategyTest.java | 170 +----------- .../ro_crate/writer/ZipWriterTest.java | 173 +----------- 4 files changed, 272 insertions(+), 528 deletions(-) create mode 100644 src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java new file mode 100644 index 00000000..d86f0f6a --- /dev/null +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java @@ -0,0 +1,251 @@ +package edu.kit.datamanager.ro_crate.writer; + +import edu.kit.datamanager.ro_crate.Crate; +import edu.kit.datamanager.ro_crate.HelpFunctions; +import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; +import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; +import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; +import net.lingala.zip4j.ZipFile; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +abstract class CrateWriterTest { + + /** + * Saves the crate with the writer fitting to this test class. + * + * @param crate the crate to save + * @param target the target path to the save location + * @throws IOException if an error occurs while saving the crate + */ + abstract protected void saveCrate(Crate crate, Path target) throws IOException; + + @Test + void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { + Path file1 = tempDir.resolve("input.txt"); + FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); + Path dirInCrate = tempDir.resolve("dir"); + FileUtils.forceMkdir(dirInCrate.toFile()); + Path dirInDirInCrate = dirInCrate.resolve("last_dir"); + FileUtils.writeStringToFile( + dirInCrate.resolve("first.txt").toFile(), + "content of first file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + dirInDirInCrate.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + dirInCrate.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = new RoCrate.RoCrateBuilder( + "Example RO-Crate", + "The RO-Crate Root Data Entity", + "2024", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(file1) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(dirInCrate) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()) + .build(); + + Path writtenCrate = tempDir.resolve("written-crate"); + this.saveCrate(roCrate, writtenCrate); + + Path folderToMakeAssertionsOn = tempDir.resolve("checkMe"); + this.ensureCrateIsExtractedIn(writtenCrate, folderToMakeAssertionsOn); + + // test if the names of the files in the crate are correct, + // when there is an ID the file should be called the same as the entity. + assertTrue( + Files.isRegularFile(folderToMakeAssertionsOn.resolve("cp7glop.ai")), + "The file should be present and have the name adjusted to the ID" + ); + assertTrue( + Files.isDirectory(folderToMakeAssertionsOn.resolve("lots_of_little_files/")), + "The directory should be present and have the name adjusted to the ID" + ); + } + + @Test + void testWriting(@TempDir Path tempDir) throws IOException { + // We need a correct directory to compare with. + // It is built manually to ensure we meet our expectations. + // Reader-writer-consistency is tested at {@link CrateReaderTest} + // TODO test javadoc + Path correctCrate = tempDir.resolve("compare_with_me"); + Path pathToFile = correctCrate.resolve("cp7glop.ai"); + Path pathToDir = correctCrate.resolve("lots_of_little_files"); + + this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + + // Now use the builder to build the same crate independently. + // The files will be reused (we need a place to take a copy from) + RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + + Path pathToZip = tempDir.resolve("written-needs_testing.zip"); + this.saveCrate(builtCrate, pathToZip); + + // extract the zip file to a temporary directory + Path extractionPath = tempDir.resolve("extracted_for_testing"); + this.ensureCrateIsExtractedIn(pathToZip, extractionPath); + // compare the extracted directory with the correct one + assertTrue(HelpFunctions.compareTwoDir( + correctCrate.toFile(), + extractionPath.toFile())); + HelpFunctions.compareCrateJsonToFileInResources( + builtCrate, + "/json/crate/fileAndDir.json"); + } + + /** + * Ensures the crate is in extracted form in the given path. + * + * @param pathToCrate the path to the crate, may not be a folder yet + * @param expectedPath the path where the crate should be in extracted form + * @throws IOException if an error occurs while extracting the crate + */ + protected void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { + try (ZipFile zf = new ZipFile(pathToCrate.toFile())) { + zf.extractAll(expectedPath.toFile().getAbsolutePath()); + } + } + + /** + * Creates a crate structure manually. + * + * @param correctCrate the path to the crate + * @param pathToFile the path to the file + * @param pathToDir the path to the directory + * @throws IOException if an error occurs while creating the crate structure + */ + protected void createManualCrateStructure(Path correctCrate, Path pathToFile, Path pathToDir) throws IOException { + FileUtils.forceMkdir(correctCrate.toFile()); + InputStream fileJson = ZipStreamStrategyTest.class + .getResourceAsStream("/json/crate/fileAndDir.json"); + Assertions.assertNotNull(fileJson); + // fill the directory with expected files and dirs + // starting with the .json of our crate + Path json = correctCrate.resolve("ro-crate-metadata.json"); + FileUtils.copyInputStreamToFile(fileJson, json.toFile()); + // create preview + PreviewGenerator.generatePreview(correctCrate.toFile().getAbsolutePath()); + // create the files and directories + FileUtils.writeStringToFile(pathToFile.toFile(), "content of Local File", Charset.defaultCharset()); + FileUtils.forceMkdir(pathToDir.toFile()); + FileUtils.writeStringToFile( + pathToDir.resolve("first.txt").toFile(), + "content of first file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("second.txt").toFile(), + "content of second file in dir", + Charset.defaultCharset()); + FileUtils.writeStringToFile( + pathToDir.resolve("third.txt").toFile(), + "content of third file in dir", + Charset.defaultCharset()); + } + + /** + * Creates a crate resembling the one we manually create in these tests. + * + * @param pathToFile the file to add + * @param pathToSubdir the directory to add + * @return the crate builder + */ + protected RoCrate.RoCrateBuilder getCrateWithFileAndDir(Path pathToFile, Path pathToSubdir) { + return new RoCrate.RoCrateBuilder( + "Example RO-Crate", + "The RO-Crate Root Data Entity", + "2024", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ) + .addDataEntity( + new FileEntity.FileEntityBuilder() + .addProperty("name", "Diagram showing trend to increase") + .addProperty("contentSize", "383766") + .addProperty("description", "Illustrator file for Glop Pot") + .setEncodingFormat("application/pdf") + .setLocationWithExceptions(pathToFile) + .setId("cp7glop.ai") + .build() + ) + .addDataEntity( + new DataSetEntity.DataSetBuilder() + .addProperty("name", "Too many files") + .addProperty("description", + "This directory contains many small files, that we're not going to describe in detail.") + .setLocationWithExceptions(pathToSubdir) + .setId("lots_of_little_files/") + .build() + ) + .setPreview(new AutomaticPreview()); + } + + @Test + void testWritingFail(@TempDir Path tempDir) throws IOException { + Path correctCrate = tempDir.resolve("compare_with_me"); + Path pathToFile = correctCrate.resolve("input.txt"); + Path pathToDir = correctCrate.resolve("dir"); + + this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + { + // another file, which we will forget in our definition + Path falseFile = tempDir.resolve("new"); + FileUtils.writeStringToFile( + falseFile.toFile(), + "this file contains something else", + Charset.defaultCharset()); + } + + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + + Path pathToZip = tempDir.resolve("writing-needs_testing.zip"); + this.saveCrate(roCrate, pathToZip); + + // extract and compare + Path extractionPath = tempDir.resolve("extracted_for_testing"); + ensureCrateIsExtractedIn(pathToZip, extractionPath); + assertFalse(HelpFunctions.compareTwoDir( + correctCrate.toFile(), + extractionPath.toFile())); + HelpFunctions.compareCrateJsonToFileInResources( + roCrate, + "/json/crate/fileAndDir.json"); + } +} diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java index ba6ecc1a..7a469465 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/FolderWriterTest.java @@ -1,214 +1,26 @@ package edu.kit.datamanager.ro_crate.writer; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.nio.file.Files; import java.nio.file.Path; -import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; -import edu.kit.datamanager.ro_crate.preview.CustomPreview; -import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; +import edu.kit.datamanager.ro_crate.Crate; import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; /** * @author Nikola Tzotchev on 9.2.2022 г. * @version 1 */ -class FolderWriterTest { - - @Test - void writeToFolderCorrectNames(@TempDir Path tempDir) throws IOException { - Path fileWithoutID = tempDir.resolve("spo.txt"); - FileUtils.writeStringToFile(fileWithoutID.toFile(), "content", Charset.defaultCharset()); - Path file1 = tempDir.resolve("input.txt"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = tempDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); +class FolderWriterTest extends CrateWriterTest { - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - Path result = tempDir.resolve("dest"); - - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - folderRoCrateWriter.save(roCrate, result.toFile().toString()); - // test if the names of the files in the crate are correct, - // when there is an ID the file should be called the same as the entity. - assertTrue(Files.isRegularFile(result.resolve("cp7glop.ai"))); - assertTrue(Files.isDirectory(result.resolve("lots_of_little_files/"))); - // assertTrue(Files.isRegularFile(result.resolve(fileWithoutID.getFileName()))); + @Override + protected void saveCrate(Crate crate, Path target) throws IOException { + Writers.newFolderWriter() + .save(crate, target.toAbsolutePath().toString()); } - - @Test - void writeToFolderTest(@TempDir Path tempDir) throws IOException { - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson - = FolderWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - //create preview via rochtml or custom as fallback - if (PreviewGenerator.isRochtmlAvailable()) { - PreviewGenerator.generatePreview(roDir.toString()); - } else { - new CustomPreview().saveAllToFolder(roDir.toFile()); - } - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("lots_of_little_files"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - Path result = tempDir.resolve("dest"); - folderRoCrateWriter.save(roCrate, result.toFile().toString()); - assertTrue(HelpFunctions.compareTwoDir(result.toFile(), roDir.toFile())); - - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } - - @Test - void writeToFolderWrongTest(@TempDir Path tempDir) throws IOException { - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson - = FolderWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - // fill the expected directory with files and dirs - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toString()); - - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("lots_of_little_files"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - // false file, this test case should fal - Path falseFile = tempDir.resolve("new"); - FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(falseFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .setId("lots_of_little_files/") - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .build(); - - Path result = tempDir.resolve("dest"); - folderRoCrateWriter.save(roCrate, result.toFile().toString()); - assertFalse(HelpFunctions.compareTwoDir(result.toFile(), roDir.toFile())); - - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + @Override + protected void ensureCrateIsExtractedIn(Path pathToCrate, Path expectedPath) throws IOException { + FileUtils.copyDirectory(pathToCrate.toFile(), expectedPath.toFile()); } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index d8ea8c6c..2ba74518 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -1,176 +1,18 @@ package edu.kit.datamanager.ro_crate.writer; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.io.*; -import java.nio.charset.Charset; import java.nio.file.Path; -import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; -import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; -import net.lingala.zip4j.ZipFile; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; +import edu.kit.datamanager.ro_crate.Crate; /** * @author jejkal */ -class ZipStreamStrategyTest { - - @Test - void testWritingToZipStream(@TempDir Path tempDir) throws IOException { - // create the RO_crate directory in the tempDir - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson= - ZipStreamStrategyTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - Assertions.assertNotNull(fileJson); - - // fill the expected directory with files and dirs - - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toFile().getAbsolutePath()); - - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - // create a Writer for writing RoCrates to zip - CrateWriter writer = Writers.newZipStreamWriter(); - // write into destination path - Path destination_path = tempDir.resolve("test.zip"); - OutputStream destination = new FileOutputStream(destination_path.toFile()); - writer.save(roCrate, destination); - - // extract and compare - Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(destination_path.toFile())) { - zf.extractAll(res.toFile().getAbsolutePath()); - } - assertTrue(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); - - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } - - - @Test - void testWritingToZipFail(@TempDir Path tempDir) throws IOException { - // create the RO_crate directory in the tempDir - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson= - ZipStreamStrategyTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - Assertions.assertNotNull(fileJson); - - // fill the expected directory with files and dirs - - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toFile().getAbsolutePath()); - - Path file1 = roDir.resolve("input.txt"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - // false file, this test case should fal - Path falseFile = tempDir.resolve("new"); - FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(falseFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - // create a Writer for writing RoCrates to zip - CrateWriter writer = Writers.newZipStreamWriter(); - // write into destination path - Path destination_path = tempDir.resolve("test.zip"); - OutputStream destination = new FileOutputStream(destination_path.toFile()); - writer.save(roCrate, destination); - - // extract and compare - Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(destination_path.toFile())) { - zf.extractAll(res.toFile().getAbsolutePath()); - } - assertFalse(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); +class ZipStreamStrategyTest extends CrateWriterTest { - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); + @Override + protected void saveCrate(Crate crate, Path target) throws IOException { + FileOutputStream stream = new FileOutputStream(target.toFile()); + Writers.newZipStreamWriter().save(crate, stream); } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java index a3a2e231..3b8b1fc0 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipWriterTest.java @@ -1,175 +1,14 @@ package edu.kit.datamanager.ro_crate.writer; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; import java.nio.file.Path; -import edu.kit.datamanager.ro_crate.HelpFunctions; -import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; -import edu.kit.datamanager.ro_crate.entities.data.FileEntity; -import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; -import edu.kit.datamanager.ro_crate.preview.CustomPreview; -import edu.kit.datamanager.ro_crate.preview.PreviewGenerator; -import net.lingala.zip4j.ZipFile; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * @author Nikola Tzotchev on 9.2.2022 г. - * @version 1 - */ -class ZipWriterTest { - - @Test - void testWritingToZip(@TempDir Path tempDir) throws IOException { - // create the RO_crate directory in the tempDir - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson - = ZipWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - //create preview via rochtml or custom as fallback - if (PreviewGenerator.isRochtmlAvailable()) { - PreviewGenerator.generatePreview(roDir.toString()); - } else { - new CustomPreview().saveAllToFolder(roDir.toFile()); - } - Path file1 = roDir.resolve("cp7glop.ai"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); - - // safe the crate in the test.zip file - Path test = tempDir.resolve("test.zip"); - // create a Writer for writing RoCrates to zip - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, test.toString()); - roCrateZipWriter.save(roCrate, "/Users/jejkal/cra.zip"); - - Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(test.toFile())) { - zf.extractAll(res.toString()); - } - assertTrue(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); - - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); - } - - @Test - void testWritingToZipFail(@TempDir Path tempDir) throws IOException { - // create the RO_crate directory in the tempDir - Path roDir = tempDir.resolve("ro_dir"); - FileUtils.forceMkdir(roDir.toFile()); - - // the .json of our crate - InputStream fileJson - = ZipWriterTest.class.getResourceAsStream("/json/crate/fileAndDir.json"); - - // fill the expected directory with files and dirs - Path json = roDir.resolve("ro-crate-metadata.json"); - FileUtils.copyInputStreamToFile(fileJson, json.toFile()); - - PreviewGenerator.generatePreview(roDir.toFile().getAbsolutePath()); - - Path file1 = roDir.resolve("input.txt"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = roDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - FileUtils.writeStringToFile(dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile(dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - // false file, this test case should fal - Path falseFile = tempDir.resolve("new"); - FileUtils.writeStringToFile(falseFile.toFile(), "this file contains something else", Charset.defaultCharset()); - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder("Example RO-Crate", - "The RO-Crate Root Data Entity", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(falseFile) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Too many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .build(); - - // safe the crate in the test.zip file - Path test = tempDir.resolve("test.zip"); - // create a Writer for writing RoCrates to zip - RoCrateWriter roCrateZipWriter = new RoCrateWriter(new ZipWriter()); - // save the content of the roCrate to the dest zip - roCrateZipWriter.save(roCrate, test.toFile().getAbsolutePath()); - Path res = tempDir.resolve("dest"); - try (ZipFile zf = new ZipFile(test.toFile())) { - zf.extractAll(res.toFile().getAbsolutePath()); - } - assertFalse(HelpFunctions.compareTwoDir(roDir.toFile(), res.toFile())); +import edu.kit.datamanager.ro_crate.Crate; - // just so we know the metadata is still valid - HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/fileAndDir.json"); +class ZipWriterTest extends CrateWriterTest { + @Override + protected void saveCrate(Crate crate, Path target) throws IOException { + Writers.newZipPathWriter() + .save(crate, target.toAbsolutePath().toString()); } } From 532880c95d25383c7250f6e8a58518b2bf698f2d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 28 Apr 2025 19:32:56 +0200 Subject: [PATCH 64/88] cleanup: remove whitespace --- .../edu/kit/datamanager/ro_crate/writer/FolderStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java index a4d1b63d..b2585637 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/FolderStrategy.java @@ -37,7 +37,7 @@ public void save(Crate crate, String destination) { File json = new File(destination, "ro-crate-metadata.json"); FileUtils.copyInputStreamToFile(inputStream, json); - inputStream.close(); + inputStream.close(); // save also the preview files to the crate destination if (crate.getPreview() != null) { crate.getPreview().saveAllToFolder(file); From 45b2be4795795e69704747f99d93408c92e58cf7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Mon, 28 Apr 2025 19:37:40 +0200 Subject: [PATCH 65/88] refactor(test): replace deprecated writer use in specific writer test --- .../ro_crate/writer/RoCrateWriterSpec12Test.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java index fc4dd016..90161e4d 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/RoCrateWriterSpec12Test.java @@ -29,11 +29,8 @@ void writeDoesNotModifyTest(@TempDir Path tempDir) throws IOException, URISyntax Crate crate = Readers.newFolderReader().readCrate(internalOriginalCrateURL.getPath()); Path targetDir = tempDir.resolve("spec12writeUnmodified"); - { - // save to disk - RoCrateWriter folderRoCrateWriter = new RoCrateWriter(new FolderWriter()); - folderRoCrateWriter.save(crate, targetDir.toFile().getPath()); - } + Writers.newFolderWriter() + .save(crate, targetDir.toAbsolutePath().toString()); // compare directories Path srcDir = Paths.get(internalOriginalCrateURL.toURI()); From d1accd8057020199f2b2b81fd162ef839d12a9af Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 12:09:04 +0200 Subject: [PATCH 66/88] refactor(test): enhance CrateWriterTest readability and output --- .../datamanager/ro_crate/HelpFunctions.java | 16 +-- .../ro_crate/writer/CrateWriterTest.java | 134 ++++++++++++------ 2 files changed, 102 insertions(+), 48 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java index 48c2d858..8f32213f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/HelpFunctions.java @@ -91,27 +91,27 @@ public static void compareCrateJsonToFileInResources(Crate crate1, String jsonFi public static boolean compareTwoDir(File dir1, File dir2) throws IOException { // compare the content of the two directories - Map result_map = FileUtils.listFiles(dir1, null, true) + Map compareWithMe = FileUtils.listFiles(dir1, null, true) .stream() .collect(Collectors.toMap(java.io.File::getName, Function.identity())); - Map input_map = FileUtils.listFiles(dir2, null, true) + Map testMe = FileUtils.listFiles(dir2, null, true) .stream() - .collect(Collectors.toMap(java.io.File::getName, Function.identity()));; + .collect(Collectors.toMap(java.io.File::getName, Function.identity())); - if (result_map.size() != input_map.size()) { + if (compareWithMe.size() != testMe.size()) { return false; } - for (String s : input_map.keySet()) { + for (String filename : testMe.keySet()) { // we do that because the ro-crate-metadata.json can be differently formatted, // or the order of the entities may be different // the same holds for the html file - if (s.equals("ro-crate-preview.html") || s.equals("ro-crate-metadata.json")) { - if (!result_map.containsKey(s)) { + if (filename.equals("ro-crate-preview.html") || filename.equals("ro-crate-metadata.json")) { + if (!compareWithMe.containsKey(filename)) { return false; } - } else if (!FileUtils.contentEqualsIgnoreEOL(input_map.get(s), result_map.get(s), null)) { + } else if (!FileUtils.contentEqualsIgnoreEOL(testMe.get(filename), compareWithMe.get(filename), null)) { return false; } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java index d86f0f6a..a5527a25 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java @@ -33,6 +33,13 @@ abstract class CrateWriterTest { */ abstract protected void saveCrate(Crate crate, Path target) throws IOException; + /** + * Test where the writer needs to rename files or folders in order to make a valid crate. + * The content will therefore not be equal to its source! + * + * @param tempDir the temporary directory given by junit for our test + * @throws IOException if an error occurs while writing the crate + */ @Test void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Path file1 = tempDir.resolve("input.txt"); @@ -85,27 +92,38 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Path writtenCrate = tempDir.resolve("written-crate"); this.saveCrate(roCrate, writtenCrate); - Path folderToMakeAssertionsOn = tempDir.resolve("checkMe"); - this.ensureCrateIsExtractedIn(writtenCrate, folderToMakeAssertionsOn); + Path extractionPath = tempDir.resolve("checkMe"); + this.ensureCrateIsExtractedIn(writtenCrate, extractionPath); + + printFileTree(extractionPath); // test if the names of the files in the crate are correct, // when there is an ID the file should be called the same as the entity. assertTrue( - Files.isRegularFile(folderToMakeAssertionsOn.resolve("cp7glop.ai")), + Files.isRegularFile(extractionPath.resolve("cp7glop.ai")), "The file should be present and have the name adjusted to the ID" ); + assertFalse( + Files.isDirectory(extractionPath.resolve("dir")), + "The directory should not be present, because 'dir' is a file in the crate" + ); assertTrue( - Files.isDirectory(folderToMakeAssertionsOn.resolve("lots_of_little_files/")), - "The directory should be present and have the name adjusted to the ID" + Files.isDirectory(extractionPath.resolve("lots_of_little_files/")), + "The directory 'lots_of_little_files' should be present and have the name adjusted to the ID" ); } + /** + * Test where the writer should make an exact copy of our defined folder. + * + * @param tempDir the temporary directory given by junit for our test + * @throws IOException if an error occurs while writing the crate + */ @Test - void testWriting(@TempDir Path tempDir) throws IOException { + void testWritingMakesCopy(@TempDir Path tempDir) throws IOException { // We need a correct directory to compare with. // It is built manually to ensure we meet our expectations. // Reader-writer-consistency is tested at {@link CrateReaderTest} - // TODO test javadoc Path correctCrate = tempDir.resolve("compare_with_me"); Path pathToFile = correctCrate.resolve("cp7glop.ai"); Path pathToDir = correctCrate.resolve("lots_of_little_files"); @@ -122,6 +140,9 @@ void testWriting(@TempDir Path tempDir) throws IOException { // extract the zip file to a temporary directory Path extractionPath = tempDir.resolve("extracted_for_testing"); this.ensureCrateIsExtractedIn(pathToZip, extractionPath); + printFileTree(correctCrate); + printFileTree(extractionPath); + // compare the extracted directory with the correct one assertTrue(HelpFunctions.compareTwoDir( correctCrate.toFile(), @@ -131,6 +152,72 @@ void testWriting(@TempDir Path tempDir) throws IOException { "/json/crate/fileAndDir.json"); } + /** + * Test where the writer should only consider the files that are added to the metadata json. + * The crate should not contain any files that are not part of it. + * + * @param tempDir the temporary directory given by junit for our test + * @throws IOException if an error occurs while writing the crate + */ + @Test + void testWritingOnlyConsidersAddedFiles(@TempDir Path tempDir) throws IOException { + Path correctCrate = tempDir.resolve("compare_with_me"); + Path pathToFile = correctCrate.resolve("cp7glop.ai"); + Path pathToDir = correctCrate.resolve("lots_of_little_files"); + + this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); + { + // This file is not part of the crate, and should therefore not be present + Path falseFile = correctCrate.resolve("new"); + FileUtils.writeStringToFile( + falseFile.toFile(), + "this file contains something else", + Charset.defaultCharset()); + } + + // create the RO_Crate including the files that should be present in it + RoCrate roCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + + Path pathToZip = tempDir.resolve("writing-needs_testing.zip"); + this.saveCrate(roCrate, pathToZip); + + + // extract and compare + Path extractionPath = tempDir.resolve("extracted_for_testing"); + ensureCrateIsExtractedIn(pathToZip, extractionPath); + printFileTree(correctCrate); + printFileTree(extractionPath); + + assertFalse(HelpFunctions.compareTwoDir( + correctCrate.toFile(), + extractionPath.toFile()), + "The crate should not contain the file that was not part of the metadata"); + HelpFunctions.compareCrateJsonToFileInResources( + roCrate, + "/json/crate/fileAndDir.json"); + } + + /** + * Prints the file tree of the given directory for debugging and understanding + * a test more quickly. + * + * @param directoryToPrint the directory to print + * @throws IOException if an error occurs while printing the file tree + */ + @SuppressWarnings("resource") + protected static void printFileTree(Path directoryToPrint) throws IOException { + // Print all files recursively in a tree structure for debugging + System.out.printf("Files in %s:%n", directoryToPrint.getFileName().toString()); + Files.walk(directoryToPrint) + .forEach(path -> { + if (!path.toAbsolutePath().equals(directoryToPrint.toAbsolutePath())) { + int depth = path.relativize(directoryToPrint).getNameCount(); + String prefix = " ".repeat(depth); + System.out.printf("%s%s%s%n", prefix, "└── ", path.getFileName()); + } + }); + } + /** * Ensures the crate is in extracted form in the given path. * @@ -215,37 +302,4 @@ protected RoCrate.RoCrateBuilder getCrateWithFileAndDir(Path pathToFile, Path pa ) .setPreview(new AutomaticPreview()); } - - @Test - void testWritingFail(@TempDir Path tempDir) throws IOException { - Path correctCrate = tempDir.resolve("compare_with_me"); - Path pathToFile = correctCrate.resolve("input.txt"); - Path pathToDir = correctCrate.resolve("dir"); - - this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); - { - // another file, which we will forget in our definition - Path falseFile = tempDir.resolve("new"); - FileUtils.writeStringToFile( - falseFile.toFile(), - "this file contains something else", - Charset.defaultCharset()); - } - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); - - Path pathToZip = tempDir.resolve("writing-needs_testing.zip"); - this.saveCrate(roCrate, pathToZip); - - // extract and compare - Path extractionPath = tempDir.resolve("extracted_for_testing"); - ensureCrateIsExtractedIn(pathToZip, extractionPath); - assertFalse(HelpFunctions.compareTwoDir( - correctCrate.toFile(), - extractionPath.toFile())); - HelpFunctions.compareCrateJsonToFileInResources( - roCrate, - "/json/crate/fileAndDir.json"); - } } From 3cfda3ae761687d9cbda185de762d6a47ca6f2ac Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 12:29:00 +0200 Subject: [PATCH 67/88] refactor(test): make testFilesBeingAdjusted understandable --- .../ro_crate/writer/CrateWriterTest.java | 87 +++++++------------ 1 file changed, 31 insertions(+), 56 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java index a5527a25..0e6e51e0 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java @@ -42,74 +42,43 @@ abstract class CrateWriterTest { */ @Test void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { - Path file1 = tempDir.resolve("input.txt"); - FileUtils.writeStringToFile(file1.toFile(), "content of Local File", Charset.defaultCharset()); - Path dirInCrate = tempDir.resolve("dir"); - FileUtils.forceMkdir(dirInCrate.toFile()); - Path dirInDirInCrate = dirInCrate.resolve("last_dir"); - FileUtils.writeStringToFile( - dirInCrate.resolve("first.txt").toFile(), - "content of first file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - dirInDirInCrate.resolve("second.txt").toFile(), - "content of second file in dir", - Charset.defaultCharset()); - FileUtils.writeStringToFile( - dirInCrate.resolve("third.txt").toFile(), - "content of third file in dir", - Charset.defaultCharset()); - - // create the RO_Crate including the files that should be present in it - RoCrate roCrate = new RoCrate.RoCrateBuilder( - "Example RO-Crate", - "The RO-Crate Root Data Entity", - "2024", - "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" - ) - .addDataEntity( - new FileEntity.FileEntityBuilder() - .addProperty("name", "Diagram showing trend to increase") - .addProperty("contentSize", "383766") - .addProperty("description", "Illustrator file for Glop Pot") - .setEncodingFormat("application/pdf") - .setLocationWithExceptions(file1) - .setId("cp7glop.ai") - .build() - ) - .addDataEntity( - new DataSetEntity.DataSetBuilder() - .addProperty("name", "Many files") - .addProperty("description", - "This directory contains many small files, that we're not going to describe in detail.") - .setLocationWithExceptions(dirInCrate) - .setId("lots_of_little_files/") - .build() - ) - .setPreview(new AutomaticPreview()) - .build(); + Path correctCrate = tempDir.resolve("compare_with_me"); + Path pathToFile = correctCrate.resolve("you-will-need-to-rename-this-file.ai"); + Path pathToDir = correctCrate.resolve("you-will-need-to-rename-this-dir"); + this.createManualCrateStructure(correctCrate, pathToFile, pathToDir); Path writtenCrate = tempDir.resolve("written-crate"); - this.saveCrate(roCrate, writtenCrate); - Path extractionPath = tempDir.resolve("checkMe"); - this.ensureCrateIsExtractedIn(writtenCrate, extractionPath); + { + RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + this.saveCrate(builtCrate, writtenCrate); + this.ensureCrateIsExtractedIn(writtenCrate, extractionPath); + } + printFileTree(correctCrate); printFileTree(extractionPath); - // test if the names of the files in the crate are correct, - // when there is an ID the file should be called the same as the entity. + // The actual file name should **not** appear in the crate + String fileName = pathToFile.getFileName().toString(); + assertFalse( + Files.isRegularFile(extractionPath.resolve(fileName)), + "The directory should not be present, because '%s' is a file in the crate".formatted(fileName) + ); + // Instead, the file should be present with the name of the ID assertTrue( Files.isRegularFile(extractionPath.resolve("cp7glop.ai")), - "The file should be present and have the name adjusted to the ID" + "The file 'cp7glop.ai' should be present and have the name adjusted to the ID" ); + // The actual directory name should **not** appear in the crate + String dirName = pathToDir.getFileName().toString(); assertFalse( - Files.isDirectory(extractionPath.resolve("dir")), - "The directory should not be present, because 'dir' is a file in the crate" + Files.isDirectory(extractionPath.resolve(dirName)), + "The directory should not be present, because '%s' is a file in the crate".formatted(dirName) ); + // Instead, the directory should be present with the name of the ID assertTrue( Files.isDirectory(extractionPath.resolve("lots_of_little_files/")), - "The directory 'lots_of_little_files' should be present and have the name adjusted to the ID" + "The directory 'lots_of_little_files' should be present" ); } @@ -252,7 +221,13 @@ protected void createManualCrateStructure(Path correctCrate, Path pathToFile, Pa PreviewGenerator.generatePreview(correctCrate.toFile().getAbsolutePath()); // create the files and directories FileUtils.writeStringToFile(pathToFile.toFile(), "content of Local File", Charset.defaultCharset()); - FileUtils.forceMkdir(pathToDir.toFile()); + // creates the directory and a subdirectory + Path subdir = pathToDir.resolve("subdir"); + FileUtils.forceMkdir(subdir.toFile()); + FileUtils.writeStringToFile( + subdir.resolve("subsubfirst.txt").toFile(), + "content of subsub file in subsubdir", + Charset.defaultCharset()); FileUtils.writeStringToFile( pathToDir.resolve("first.txt").toFile(), "content of first file in dir", From 2cbe30ee29c93634f2f6a24cbbda8eb210300f7f Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 13:02:40 +0200 Subject: [PATCH 68/88] refactor(ZipUtil): improve method documentation and formatting --- .../datamanager/ro_crate/util/ZipUtil.java | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java index 36841318..c1da0137 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/util/ZipUtil.java @@ -13,9 +13,24 @@ */ public class ZipUtil { - public static void addFolderToZipStream(ZipOutputStream zipOutputStream, File folder, String parentPath) throws IOException { + /** + * Adds a folder and its contents to a ZipOutputStream. + * + * @param zipOutputStream The ZipOutputStream to which the folder will be added. + * @param folder The folder to be added. + * @param parentPath The path in the zip file where the folder will be added. + * @throws IOException If an I/O error occurs. + */ + public static void addFolderToZipStream( + ZipOutputStream zipOutputStream, + File folder, + String parentPath + ) throws IOException { if (!folder.exists() || !folder.isDirectory()) { - throw new IllegalArgumentException("The provided folder path is not a valid directory: " + folder.getAbsolutePath()); + throw new IllegalArgumentException( + "The provided folder path is not a valid directory: %s" + .formatted(folder.getAbsolutePath()) + ); } File[] files = folder.listFiles(); @@ -33,11 +48,34 @@ public static void addFolderToZipStream(ZipOutputStream zipOutputStream, File fo } } - public static void addFolderToZipStream(ZipOutputStream zipOutputStream, String folderPath, String parentPath) throws IOException { + /** + * Adds a folder and its contents to a ZipOutputStream. + * @param zipOutputStream The ZipOutputStream to which the folder will be added. + * @param folderPath The path of the folder to be added. + * @param parentPath The path in the zip file where the folder will be added. + * @throws IOException If an I/O error occurs. + */ + public static void addFolderToZipStream( + ZipOutputStream zipOutputStream, + String folderPath, + String parentPath + ) throws IOException { addFolderToZipStream(zipOutputStream, new File(folderPath), parentPath); } - public static void addFileToZipStream(ZipOutputStream zipOutputStream, File file, String zipEntryPath) throws IOException { + /** + * Adds a file to a ZipOutputStream. + * + * @param zipOutputStream The ZipOutputStream to which the file will be added. + * @param file The file to be added. + * @param zipEntryPath The path in the zip file where the file will be added. + * @throws IOException If an I/O error occurs. + */ + public static void addFileToZipStream( + ZipOutputStream zipOutputStream, + File file, + String zipEntryPath + ) throws IOException { ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip(zipEntryPath); zipOutputStream.putNextEntry(zipParameters); From 378b81e18fff0526f220ba380f767823cfc05b6d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 13:03:16 +0200 Subject: [PATCH 69/88] test(CrateWriterTest): verify presence of subdirectory in extraction path --- .../edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java index 0e6e51e0..41ebedec 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java @@ -80,6 +80,10 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Files.isDirectory(extractionPath.resolve("lots_of_little_files/")), "The directory 'lots_of_little_files' should be present" ); + assertTrue( + Files.isDirectory(extractionPath.resolve("lots_of_little_files/").resolve("subdir")), + "The directory 'lots_of_little_files/subdir' should be present" + ); } /** From a81f8f8556006117e793dbbd9dc354042e1e4866 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 13:05:37 +0200 Subject: [PATCH 70/88] fix(zip writers): ensure the dataset also sets its name properly in tests --- .../ro_crate/entities/data/DataSetEntity.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java index 97d302f7..832d9819 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataSetEntity.java @@ -12,6 +12,7 @@ import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.outputstream.ZipOutputStream; +import net.lingala.zip4j.model.ZipParameters; /** * A helping class for the creating of Data entities of type Dataset. @@ -45,14 +46,20 @@ public void removeFromHasPart(String str) { @Override public void saveToZip(ZipFile zipFile) throws ZipException { if (this.getPath() != null) { - zipFile.addFolder(this.getPath().toFile()); + ZipParameters parameters = new ZipParameters(); + parameters.setRootFolderNameInZip(this.getId()); + parameters.setIncludeRootFolder(false); + zipFile.addFolder(this.getPath().toFile(), parameters); } } @Override - public void saveToStream(ZipOutputStream zipOutputStream) throws ZipException, IOException { + public void saveToStream(ZipOutputStream zipOutputStream) throws IOException { if (this.getPath() != null) { - ZipUtil.addFolderToZipStream(zipOutputStream, this.getPath().toAbsolutePath().toString(), this.getPath().getFileName().toString()); + ZipUtil.addFolderToZipStream( + zipOutputStream, + this.getPath().toAbsolutePath().toString(), + this.getId()); } } From c14350c06df2faf38e667526b5256aff6dca88a8 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 13:30:56 +0200 Subject: [PATCH 71/88] test(CrateWriterTest): test and document subdirectory behavior --- .../ro_crate/entities/data/DataEntity.java | 8 ++++++-- .../ro_crate/writer/CrateWriterTest.java | 20 ++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 50ca8d2a..65c7f928 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -117,8 +117,12 @@ abstract static class AbstractDataEntityBuilder authors = new ArrayList<>(); /** - * Sets the location of the data entity. - * + * Sets the location of the data entity to make a copy from when writing the crate. + *

+ * Use this if you want to copy the associated file to the crate. + * Do not use it if the file should already be considered (e.g. + * as part of a DataSetEntity). + *

* If the ID has not been set manually in beforehand, it will be derived * from the path. Use {@link #setId(String)} to override it or set it in * beforehand. Note that another call of {@link #setLocation(Path)} will diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java index 41ebedec..429fc450 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CrateWriterTest.java @@ -50,7 +50,15 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Path writtenCrate = tempDir.resolve("written-crate"); Path extractionPath = tempDir.resolve("checkMe"); { - RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir).build(); + RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir) + .addDataEntity(new DataSetEntity.DataSetBuilder() + .addProperty("name", "Subdir") + .addProperty("description", "Some subdir") + .setLocationWithExceptions(pathToDir.resolve("subdir")) + .setId("lots_of_little_files/subdir-renamed/") + .build() + ) + .build(); this.saveCrate(builtCrate, writtenCrate); this.ensureCrateIsExtractedIn(writtenCrate, extractionPath); } @@ -84,6 +92,16 @@ void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Files.isDirectory(extractionPath.resolve("lots_of_little_files/").resolve("subdir")), "The directory 'lots_of_little_files/subdir' should be present" ); + + /* + * As we added another DataSetEntity with location, the subdir should have a renamed copy as well + * Note: If this behavior is to be changed later on, we possibly need to change the documentation + * of {@link AbstractDataEntityBuilder#setLocation(Path)}. + */ + assertTrue( + Files.isDirectory(extractionPath.resolve("lots_of_little_files/").resolve("subdir-renamed")), + "The directory 'lots_of_little_files/subdir' should be present" + ); } /** From 1ce985ca78970063871a1e9bce110cd02c9a634b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 13:33:26 +0200 Subject: [PATCH 72/88] refactor(DataEntity): improve Javadoc formatting and remove unused imports --- .../ro_crate/entities/data/DataEntity.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 65c7f928..e6e28f8f 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -8,9 +8,7 @@ import edu.kit.datamanager.ro_crate.util.ZipUtil; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Path; import java.util.ArrayList; @@ -48,11 +46,12 @@ public DataEntity(AbstractDataEntityBuilder entityBuilder) { /** * Adds an author ID to the entity. - * + *

* Calling this multiple times will add multiple author IDs. * * @param id the identifier of the author. */ + @SuppressWarnings("unused") public void addAuthorId(String id) { this.addIdProperty("author", id); } @@ -125,7 +124,7 @@ abstract static class AbstractDataEntityBuilder * If the ID has not been set manually in beforehand, it will be derived * from the path. Use {@link #setId(String)} to override it or set it in - * beforehand. Note that another call of {@link #setLocation(Path)} will + * beforehand. Note that another call of this method will * not override the ID as it has been set by the previous call! * * @param path the location of the data. May be null, in which case @@ -159,7 +158,7 @@ public T setLocationWithExceptions(Path path) throws IllegalArgumentException { /** * Same as {@link #setLocation(Path)} but instead of associating this * entity with a file, it will point to some place on the internet. - * + *

* Via the specification, this means the uri will be set as the ID. This * call is therefore equivalent to {@link #setId(String)}. * @@ -214,7 +213,7 @@ public T setContentLocation(String id) { /** * Data Entity builder class that allows for easier data entity creation. - * + *

* If not explicitly mentioned, all methods avoid Exceptions and will * silently ignore null-parameters, in which case nothing will happen. Use * the available *WithExceptions-methods in case you need them. From 14ebbd3747a83cb397682a509f63a14711f16478 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 14:22:44 +0200 Subject: [PATCH 73/88] refactor(RorProvider): replace get with path for safer JSON access --- .../organizationprovider/RorProvider.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java index 3885c4b1..d12f3a7c 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java @@ -53,10 +53,10 @@ public static OrganizationEntity getOrganization(String url) { ObjectNode.class); return new OrganizationEntity.OrganizationEntityBuilder() - .setId(jsonNode.get("id").asText()) - .addProperty("name", getOrganizationNameV2(jsonNode.get("names"))) - .addProperty("email", jsonNode.get("email_address")) - .addProperty("url", jsonNode.get("id").asText()) + .setId(jsonNode.path("id").asText()) + .addProperty("name", getOrganizationNameV2(jsonNode.path("names"))) + .addProperty("email", jsonNode.path("email_address")) + .addProperty("url", jsonNode.path("id").asText()) .build(); } catch (IOException e) { String errorMessage = String.format("IO error: %s", e.getMessage()); @@ -68,15 +68,15 @@ public static OrganizationEntity getOrganization(String url) { private static String getOrganizationNameV2(JsonNode node) { if (node.isArray()) { for (JsonNode n : node) { - if (n.has("types") && n.get("types").isArray()) { - ArrayList l = new ObjectMapper().convertValue(n.get("types"), ArrayList.class); + if (n.has("types") && n.path("types").isArray()) { + ArrayList l = new ObjectMapper().convertValue(n.path("types"), ArrayList.class); if (l.contains("ror_display")) { - return n.get("value").asText(); + return n.path("value").asText(); } } } //fallback - return node.get(0).get("value").asText(); + return node.path(0).path("value").asText(); } else { return node.asText(); } From 8bffc903243b41d1baadd9d72e09b039ae6678ba Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 14:23:51 +0200 Subject: [PATCH 74/88] cleanup: formatting and smaller linter proposals --- .../externalproviders/organizationprovider/RorProvider.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java index d12f3a7c..350fdad6 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/externalproviders/organizationprovider/RorProvider.java @@ -23,7 +23,7 @@ */ public class RorProvider { - private static Logger logger = LoggerFactory.getLogger(RorProvider.class); + private static final Logger logger = LoggerFactory.getLogger(RorProvider.class); private RorProvider() { } @@ -42,7 +42,7 @@ public static OrganizationEntity getOrganization(String url) { HttpGet request = new HttpGet(newUrl); try ( - CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpResponse response = httpClient.execute(request);) { + CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpResponse response = httpClient.execute(request)) { boolean isError = response.getStatusLine().getStatusCode() != HttpStatus.SC_OK; if (isError) { String errorMessage = String.format("Identifier not found: %s", response.getStatusLine().toString()); @@ -69,7 +69,7 @@ private static String getOrganizationNameV2(JsonNode node) { if (node.isArray()) { for (JsonNode n : node) { if (n.has("types") && n.path("types").isArray()) { - ArrayList l = new ObjectMapper().convertValue(n.path("types"), ArrayList.class); + ArrayList l = new ObjectMapper().convertValue(n.path("types"), ArrayList.class); if (l.contains("ror_display")) { return n.path("value").asText(); } From 2f54be887205b46eea8ea30ea728927ba4cb7f42 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 14:36:59 +0200 Subject: [PATCH 75/88] refactor(ZipStrategy): use try-with-resources for InputStream management --- .../edu/kit/datamanager/ro_crate/writer/ZipStrategy.java | 8 ++++---- .../ro_crate/writer/ZipStreamStrategyTest.java | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index c944aad3..d64efdac 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -54,10 +54,10 @@ private void saveMetadataJson(Crate crate, ZipFile zipFile) { // we create an JsonNode only to have the file written pretty JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); - // write the ro-crate-metadata - zipFile.addStream(inputStream, zipParameters); - inputStream.close(); + try (InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { + // write the ro-crate-metadata + zipFile.addStream(inputStream, zipParameters); + } if (crate.getPreview() != null) { crate.getPreview().saveAllToZip(zipFile); } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java index 2ba74518..283b306a 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/ZipStreamStrategyTest.java @@ -12,7 +12,8 @@ class ZipStreamStrategyTest extends CrateWriterTest { @Override protected void saveCrate(Crate crate, Path target) throws IOException { - FileOutputStream stream = new FileOutputStream(target.toFile()); - Writers.newZipStreamWriter().save(crate, stream); + try (FileOutputStream stream = new FileOutputStream(target.toFile())) { + Writers.newZipStreamWriter().save(crate, stream); + } } } From 0f30a145cee48853c21af95eeb497d69de7fa77a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 14:46:45 +0200 Subject: [PATCH 76/88] cleanup(ZipStrategy): do not put everything into the try block --- .../edu/kit/datamanager/ro_crate/writer/ZipStrategy.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java index d64efdac..6a11df87 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/ZipStrategy.java @@ -46,11 +46,11 @@ private void saveDataEntities(Crate crate, ZipFile zipFile) { } private void saveMetadataJson(Crate crate, ZipFile zipFile) { + // write the metadata.json file + ZipParameters zipParameters = new ZipParameters(); + zipParameters.setFileNameInZip("ro-crate-metadata.json"); + ObjectMapper objectMapper = MyObjectMapper.getMapper(); try { - // write the metadata.json file - ZipParameters zipParameters = new ZipParameters(); - zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ObjectMapper objectMapper = MyObjectMapper.getMapper(); // we create an JsonNode only to have the file written pretty JsonNode node = objectMapper.readTree(crate.getJsonMetadata()); String str = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); From cafe69619ac4e7b69af1e8fedc09e1fab6193d96 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 14:47:04 +0200 Subject: [PATCH 77/88] refactor(CrateReader): improve file resolution with path normalization and safety checks --- .../edu/kit/datamanager/ro_crate/reader/CrateReader.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 35fe0f66..9b96bf5a 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -224,7 +225,10 @@ protected Optional checkFolderHasFile(String filepathOrId, File folder) { return Optional.empty(); } return IdentifierUtils.decode(filepathOrId) - .map(decoded -> folder.toPath().resolve(decoded).toFile()) + .map(decoded -> folder.toPath().resolve(decoded).normalize()) + // defence-in-depth: ensure we are still inside the crate folder + .filter(resolved -> resolved.startsWith(folder.toPath())) + .map(Path::toFile) .filter(File::exists); } From 3f2c1cb6f7d85dc3071b1c0496ebafbcc56c2dac Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 14:52:19 +0200 Subject: [PATCH 78/88] refactor(CrateReader): enforce non-null checks for metadataJson and files parameters --- .../kit/datamanager/ro_crate/reader/CrateReader.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 9b96bf5a..5acb575b 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -98,14 +98,10 @@ public RoCrate readCrate(T location) { } private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet usedFiles) { - if (metadataJson == null) { - logger.error("Metadata JSON is null, cannot rebuild crate"); - return null; - } - if (files == null) { - logger.error("Content files directory is null, cannot rebuild crate"); - return null; - } + Objects.requireNonNull(metadataJson, + "metadataJson must not be null – did the strategy fail to locate 'ro-crate-metadata.json'?"); + Objects.requireNonNull(files, + "files directory must not be null – check GenericReaderStrategy.readContent()"); JsonNode context = metadataJson.get(PROP_CONTEXT); CrateMetadataContext crateContext = new RoCrateMetadataContext(context); From e71f132aa5ccfa724ce4c6968c00368112e35dcc Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 16:08:39 +0200 Subject: [PATCH 79/88] cleanup: delete JsonLdExpander.java --- .../ro_crate/util/JsonLdExpander.java | 116 ------------------ 1 file changed, 116 deletions(-) delete mode 100644 src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java diff --git a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java b/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java deleted file mode 100644 index 05622a97..00000000 --- a/src/main/java/edu/kit/datamanager/ro_crate/util/JsonLdExpander.java +++ /dev/null @@ -1,116 +0,0 @@ -package edu.kit.datamanager.ro_crate.util; - -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.databind.node.*; - -import java.util.*; - -import java.io.File; - -/** - * Utility class for expanding and pruning JSON-LD documents. - *

- * This class provides functionality to resolve references in JSON-LD based on "@id" values, - * expanding the data structure by replacing references with their full content. - * It also handles circular references to prevent infinite recursion. - * - * @author jejkal - */ -public class JsonLdExpander { - - public static JsonNode expandAndPrune(File jsonLdFile) throws Exception { - if (jsonLdFile == null) { - throw new IllegalArgumentException("JSON-LD file must not be null."); - } - if (!jsonLdFile.exists() || !jsonLdFile.canRead()) { - throw new IllegalArgumentException("JSON-LD file does not exist or cannot be read: " + jsonLdFile.getAbsolutePath()); - } - ObjectMapper mapper = new ObjectMapper(); - JsonNode root = mapper.readTree(jsonLdFile); - return expandAndPrune(root); - } - - public static JsonNode expandAndPrune(JsonNode root) throws IllegalArgumentException { - if (root == null) { - throw new IllegalArgumentException("Root node must not be null."); - } - JsonNode graph = root.path("@graph"); - ObjectMapper mapper = new ObjectMapper(); - // Index all items by @id - Map idMap = new HashMap<>(); - for (JsonNode node : graph) { - if (node.has("@id")) { - idMap.put(node.path("@id").asText(), node); - } - } - - // Track which ids were directly referenced and expanded - Set expandedIds = new HashSet<>(); - - ArrayNode expandedGraph = mapper.createArrayNode(); - for (JsonNode node : graph) { - // Include only if it's NOT a referenced/expanded object - if (node.has("@id") && expandedIds.contains(node.path("@id").asText())) { - continue; // skip referenced/expanded nodes - } - - JsonNode expandedNode = expandNode(node, idMap, expandedIds, mapper, new HashSet<>()); - expandedGraph.add(expandedNode); - } - - // Rebuild root - ObjectNode newRoot = mapper.createObjectNode(); - newRoot.set("@context", root.path("@context")); - newRoot.set("@graph", expandedGraph); - return newRoot; - } - - private static JsonNode expandNode(JsonNode node, Map idMap, Set expandedIds, ObjectMapper mapper, Set visited) { - if (!node.isObject()) { - return node; - } - - ObjectNode result = mapper.createObjectNode(); - Iterator> fields = node.fields(); - - while (fields.hasNext()) { - Map.Entry entry = fields.next(); - String key = entry.getKey(); - JsonNode value = entry.getValue(); - - if (value.isObject() && value.has("@id")) { - String refId = value.path("@id").asText(); - if (!visited.contains(refId) && idMap.containsKey(refId)) { - visited.add(refId); - expandedIds.add(refId); - result.set(key, expandNode(idMap.getOrDefault(refId, NullNode.getInstance()), idMap, expandedIds, mapper, new HashSet<>(visited))); - } else { - result.set(key, value); - } - } else if (value.isArray()) { - ArrayNode newArray = mapper.createArrayNode(); - for (JsonNode element : value) { - if (element.isObject() && element.has("@id")) { - String refId = element.path("@id").asText(); - if (!visited.contains(refId) && idMap.containsKey(refId)) { - visited.add(refId); - expandedIds.add(refId); - newArray.add(expandNode(idMap.getOrDefault(refId, NullNode.getInstance()), idMap, expandedIds, mapper, new HashSet<>(visited))); - } else { - newArray.add(element); - } - } else { - newArray.add(expandNode(element, idMap, expandedIds, mapper, visited)); - } - } - result.set(key, newArray); - } else if (value.isObject()) { - result.set(key, expandNode(value, idMap, expandedIds, mapper, visited)); - } else { - result.set(key, value); - } - } - - return result; - } -} From 05d1edfed256514d1f1a01eeee9fd26af159ffbe Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 16:09:42 +0200 Subject: [PATCH 80/88] feat: add Javadoc generation step to CI pipeline --- .github/workflows/gradle.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 6efca362..eadcb900 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -47,3 +47,18 @@ jobs: - name: Do one Coveralls test report if: matrix.os == 'ubuntu-latest' && matrix.jdk == 21 run: ./gradlew -Dprofile=release jacocoTestReport coveralls + javadoc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up openJDK version + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: 'zulu' + - name: Install Dependencies + run: npm install -g ro-crate-html-js + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + - name: Compile Javadoc + run: ./gradlew javadoc \ No newline at end of file From d7bc03382d07caee67029769da97ba4c18ca0f3a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 16:28:51 +0200 Subject: [PATCH 81/88] refactor: use new writer APIs in all remaining tests --- .../ro_crate/crate/OtherFilesTest.java | 7 +++--- .../ro_crate/crate/ReadAndWriteTest.java | 8 +++---- .../crate/preview/PreviewCrateTest.java | 22 +++++++++---------- .../crate/realexamples/WorkflowHubTest.java | 7 +++--- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/OtherFilesTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/OtherFilesTest.java index b614322c..aeb275e9 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/OtherFilesTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/OtherFilesTest.java @@ -2,9 +2,8 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -41,8 +40,8 @@ void testOtherFiles(@TempDir Path tempDir) throws IOException, URISyntaxExceptio HelpFunctions.compareCrateJsonToFileInResources(roCrate, "/json/crate/simple.json"); // write the crate in the temp dir - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(roCrate, crate.toFile().getAbsolutePath()); + Writers.newFolderWriter() + .save(roCrate, crate.toFile().getAbsolutePath()); HelpFunctions.compareCrateJsonToFileInResources(new File(Objects.requireNonNull(OtherFilesTest.class.getResource("/json/crate/simple.json")).toURI()), crate.resolve("ro-crate-metadata.json").toFile()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java index 4a39cca3..3c2e49b5 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java @@ -6,9 +6,8 @@ import edu.kit.datamanager.ro_crate.preview.StaticPreview; import edu.kit.datamanager.ro_crate.reader.CrateReader; import edu.kit.datamanager.ro_crate.reader.Readers; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -36,8 +35,8 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { Path writeDir = path.resolve("crate"); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(crate, writeDir.toAbsolutePath().toString()); + Writers.newFolderWriter() + .save(crate, writeDir.toAbsolutePath().toString()); CrateReader reader = Readers.newFolderReader(); Crate newCrate = reader.readCrate(writeDir.toAbsolutePath().toString()); @@ -48,6 +47,7 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { HelpFunctions.compareTwoCrateJson(newCrate, crate); } + @SuppressWarnings("DataFlowIssue") @Test void testReadCrateWithHasPartHierarchy() { CrateReader reader = Readers.newFolderReader(); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java index 64888f08..1ecc2062 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/preview/PreviewCrateTest.java @@ -1,5 +1,7 @@ package edu.kit.datamanager.ro_crate.crate.preview; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -8,8 +10,6 @@ import edu.kit.datamanager.ro_crate.preview.AutomaticPreview; import edu.kit.datamanager.ro_crate.preview.CustomPreview; import edu.kit.datamanager.ro_crate.preview.StaticPreview; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; import java.io.IOException; import java.nio.charset.Charset; @@ -27,8 +27,8 @@ void testAutomaticPreview(@TempDir Path temp) { RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new AutomaticPreview()) .build(); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(crate, location.toFile().getAbsolutePath()); + Writers.newFolderWriter() + .save(crate, location.toFile().getAbsolutePath()); assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); } @@ -38,7 +38,7 @@ void testAutomaticPreviewAddingLater(@TempDir Path temp) { RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(null)//disable preview to allow to compare folders before and after .build(); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); + CrateWriter writer = Writers.newFolderWriter(); writer.save(crate, location.toFile().toString()); assertFalse(location.resolve("ro-crate-preview.html").toFile().exists()); crate.setRoCratePreview(new AutomaticPreview()); @@ -52,8 +52,8 @@ void testCustomPreview(@TempDir Path temp) { RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new CustomPreview()) .build(); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(crate, location.toFile().getAbsolutePath()); + Writers.newFolderWriter() + .save(crate, location.toFile().getAbsolutePath()); assertTrue(Files.isRegularFile(location.resolve("ro-crate-preview.html"))); } @@ -65,8 +65,8 @@ void testStaticPreviewOnlyHtmlFile(@TempDir Path temp) throws IOException { RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new StaticPreview(previewFile.toFile())) .build(); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(crate, location.toFile().toString()); + Writers.newFolderWriter() + .save(crate, location.toFile().getAbsolutePath()); assertTrue(location.resolve("ro-crate-preview.html").toFile().exists()); } @@ -81,8 +81,8 @@ void testStaticPreviewHtmlFileWithOtherFiles(@TempDir Path temp) throws IOExcept RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") .setPreview(new StaticPreview(previewFile.toFile(), dirHtml.toFile())) .build(); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(crate, location.toFile().toString()); + Writers.newFolderWriter() + .save(crate, location.toFile().getAbsolutePath()); assertTrue(location.resolve("ro-crate-preview.html").toFile().exists()); assertTrue(location.resolve("ro-crate-preview_files").toFile().exists()); assertTrue(location.resolve("ro-crate-preview_files").resolve("test.css").toFile().exists()); diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java index 1d45bd09..99b2b563 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/realexamples/WorkflowHubTest.java @@ -4,9 +4,8 @@ import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.reader.CrateReader; import edu.kit.datamanager.ro_crate.reader.Readers; -import edu.kit.datamanager.ro_crate.writer.FolderWriter; -import edu.kit.datamanager.ro_crate.writer.RoCrateWriter; +import edu.kit.datamanager.ro_crate.writer.Writers; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -16,14 +15,14 @@ public class WorkflowHubTest { + @SuppressWarnings("DataFlowIssue") @Test void testImportZip(@TempDir Path temp) throws IOException { CrateReader reader = Readers.newZipPathReader(); Crate crate = reader.readCrate(WorkflowHubTest.class.getResource("/crates/workflowhub/workflow-109-5.crate.zip").getPath()); HelpFunctions.compareCrateJsonToFileInResources(crate, "/crates/workflowhub/workflow1/ro-crate-metadata.json"); - RoCrateWriter writer = new RoCrateWriter(new FolderWriter()); - writer.save(crate, temp.toString()); + Writers.newFolderWriter().save(crate, temp.toString()); HelpFunctions.compareTwoDir(temp.toFile(), new File(WorkflowHubTest.class.getResource("/crates/workflowhub/workflow1/").getPath())); } } From fae0067c9c43f63c1021a62881d23dd85aae9ee7 Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 16:58:26 +0200 Subject: [PATCH 82/88] fix: change cfg to local variable in CustomPreview constructor --- .../edu/kit/datamanager/ro_crate/preview/CustomPreview.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java index 4b1349b4..5c8c9fa2 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/preview/CustomPreview.java @@ -37,11 +37,10 @@ public class CustomPreview implements CratePreview { private final static Logger logger = LoggerFactory.getLogger(CustomPreview.class); - private final Configuration cfg; private Template template = null; public CustomPreview() { - cfg = new Configuration(Configuration.VERSION_2_3_34); + Configuration cfg = new Configuration(Configuration.VERSION_2_3_34); cfg.setClassForTemplateLoading(CustomPreview.class, "/"); cfg.setDefaultEncoding("UTF-8"); cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); From 8fe411d35a43f59e22e7f318875594d1cdd84d2b Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 16:58:53 +0200 Subject: [PATCH 83/88] fix: update test cases to use more complex crates --- .../ro_crate/preview/PreviewTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java index 2ef45737..2db987b3 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/preview/PreviewTest.java @@ -21,7 +21,7 @@ public class PreviewTest { @Test void staticPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { var file1 = dir.resolve("file.html"); - FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + FileUtils.writeStringToFile(file1.toFile(), "random html, does not need to be valid for this test", Charset.defaultCharset()); var file2 = dir.resolve("directory"); var fileInDir = file2.resolve("fileInDir.html"); @@ -49,7 +49,7 @@ void staticPreviewSaveToFolderTest(@TempDir Path dir) throws IOException { @Test void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { var file1 = dir.resolve("file.html"); - FileUtils.writeStringToFile(file1.toFile(), "random html it is not important that it is valid for know", Charset.defaultCharset()); + FileUtils.writeStringToFile(file1.toFile(), "random html, does not need to be valid for this test", Charset.defaultCharset()); var file2 = dir.resolve("directory"); var fileInDir = file2.resolve("fileInDir.html"); @@ -79,7 +79,7 @@ void staticPreviewSaveToZip(@TempDir Path dir) throws IOException { void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); // this crate will not have a json file FileUtils.forceMkdir(crate.toFile()); @@ -93,7 +93,7 @@ void testAutomaticPreviewAddToFolder(@TempDir Path dir) throws IOException { @Test void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { AutomaticPreview automaticPreview = new AutomaticPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); @@ -120,7 +120,7 @@ void testAutomaticPreviewZip(@TempDir Path dir) throws IOException { void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { CustomPreview customPreview = new CustomPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); Path crate = dir.resolve("crate"); // this crate will not have a json file Path fakeCrate = dir.resolve("fakeCrate"); @@ -142,14 +142,14 @@ void testCustomPreviewAddToFolder(@TempDir Path dir) throws IOException { } @Test - void testCustomPreviewZip(@TempDir Path dir) throws IOException { + void testCustomPreviewZip(@TempDir Path tmp) throws IOException { CustomPreview customPreview = new CustomPreview(); - InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/simple_crate/ro-crate-metadata.json"); - Path crate = dir.resolve("crate"); + InputStream crateJson = PreviewTest.class.getResourceAsStream("/crates/other/idrc_project/ro-crate-metadata.json"); + Path crate = tmp.resolve("crate"); ZipParameters zipParameters = new ZipParameters(); zipParameters.setFileNameInZip("ro-crate-metadata.json"); - ZipFile zipFile = new ZipFile(dir.resolve("test.zip").toFile()); + ZipFile zipFile = new ZipFile(tmp.resolve("test.zip").toFile()); zipFile.addStream(crateJson, zipParameters); crateJson.close(); @@ -157,7 +157,7 @@ void testCustomPreviewZip(@TempDir Path dir) throws IOException { try { // this should trow an exception but not stop the execution - ZipFile randomZipFile = new ZipFile(dir.resolve("dddd.zip").toFile()); + ZipFile randomZipFile = new ZipFile(tmp.resolve("dddd.zip").toFile()); customPreview.saveAllToZip(randomZipFile); Assertions.fail("Expected IOException when providing invalid input to preview."); } catch (IOException ex) { From 8b4e4b418142e199d9d5afaff5988db67be00f1e Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 20:03:11 +0200 Subject: [PATCH 84/88] fix: wrong indentation in CI yaml --- .github/workflows/gradle.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index eadcb900..df40ed95 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -50,15 +50,15 @@ jobs: javadoc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Set up openJDK version - uses: actions/setup-java@v4 - with: - java-version: 21 - distribution: 'zulu' - - name: Install Dependencies - run: npm install -g ro-crate-html-js - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - - name: Compile Javadoc - run: ./gradlew javadoc \ No newline at end of file + - uses: actions/checkout@v4 + - name: Set up openJDK version + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: 'zulu' + - name: Install Dependencies + run: npm install -g ro-crate-html-js + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + - name: Compile Javadoc + run: ./gradlew javadoc \ No newline at end of file From 880df80f99fbe9ce7eb4a67866fc2938139cca3a Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 20:27:42 +0200 Subject: [PATCH 85/88] fix: add dependency on build step for Javadoc generation in CI --- .github/workflows/gradle.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index df40ed95..60af4073 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -48,6 +48,7 @@ jobs: if: matrix.os == 'ubuntu-latest' && matrix.jdk == 21 run: ./gradlew -Dprofile=release jacocoTestReport coveralls javadoc: + needs: [build] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From b6614ad74cf81550fe7fdb7297a4a08818dd4ebe Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 20:41:05 +0200 Subject: [PATCH 86/88] cleanup: remove unused json test files --- .../json/entities/contextual/contactPoint.json | 7 ------- src/test/resources/json/entities/contextual/geo.json | 10 ---------- .../resources/json/entities/contextual/rorkit.json | 6 ------ 3 files changed, 23 deletions(-) delete mode 100644 src/test/resources/json/entities/contextual/contactPoint.json delete mode 100644 src/test/resources/json/entities/contextual/geo.json delete mode 100644 src/test/resources/json/entities/contextual/rorkit.json diff --git a/src/test/resources/json/entities/contextual/contactPoint.json b/src/test/resources/json/entities/contextual/contactPoint.json deleted file mode 100644 index 457e18f9..00000000 --- a/src/test/resources/json/entities/contextual/contactPoint.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "@id": "mailto:tim.luckett@uts.edu.au", - "@type": "ContactPoint", - "contactType": "customer service", - "email": "tim.luckett@uts.edu.au", - "telephone": "0999444" -} \ No newline at end of file diff --git a/src/test/resources/json/entities/contextual/geo.json b/src/test/resources/json/entities/contextual/geo.json deleted file mode 100644 index 6ac07e05..00000000 --- a/src/test/resources/json/entities/contextual/geo.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "@id": "#b4168a98-8534-4c6d-a568-64a55157b656", - "@type": "GeoCoordinates", - "latitude": "-33.7152", - "longitude": "150.30119", - "name": "latitude: -33.7152 longitude: 150.30119", - "address": "this land nomero 2", - "postalcode": "76588", - "elevation": "100" -} \ No newline at end of file diff --git a/src/test/resources/json/entities/contextual/rorkit.json b/src/test/resources/json/entities/contextual/rorkit.json deleted file mode 100644 index c29ceb2a..00000000 --- a/src/test/resources/json/entities/contextual/rorkit.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "@id" : "https://ror.org/04t3en479", - "@type" : "Organization", - "name" : "Karlsruhe Institute of Technology", - "url" : "https://www.kit.edu" -} \ No newline at end of file From bda49bc4469ad0cd7e7f95be279157af417df7fc Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 20:44:32 +0200 Subject: [PATCH 87/88] fix: add newline at end of gradle.yml for consistency --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 60af4073..5d53a70c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -62,4 +62,4 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 - name: Compile Javadoc - run: ./gradlew javadoc \ No newline at end of file + run: ./gradlew javadoc From 521ca559e0330ecfe29796b1fe424ae2c19b794d Mon Sep 17 00:00:00 2001 From: Andreas Pfeil Date: Tue, 29 Apr 2025 20:46:17 +0200 Subject: [PATCH 88/88] fix: remove unnecessary Javadoc build step from CI configuration --- .github/workflows/gradle.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 5d53a70c..ba8a9308 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -47,19 +47,6 @@ jobs: - name: Do one Coveralls test report if: matrix.os == 'ubuntu-latest' && matrix.jdk == 21 run: ./gradlew -Dprofile=release jacocoTestReport coveralls - javadoc: - needs: [build] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up openJDK version - uses: actions/setup-java@v4 - with: - java-version: 21 - distribution: 'zulu' - - name: Install Dependencies - run: npm install -g ro-crate-html-js - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - name: Compile Javadoc + if: matrix.os == 'ubuntu-latest' && matrix.jdk == 21 run: ./gradlew javadoc